diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.test.tsx b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.test.tsx index c88918041ca81..675f2dec86145 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.test.tsx +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.test.tsx @@ -36,8 +36,8 @@ const mockIndexPatternCreationType = new IndexPatternCreationConfig({ }); jest.mock('../../lib/get_indices', () => ({ - getIndices: ({}, {}, query: string) => { - if (query.startsWith('e')) { + getIndices: ({ pattern }: { pattern: string }) => { + if (pattern.startsWith('e')) { return [{ name: 'es', item: {} }]; } diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx index d8555d71d6ec0..386514de68d4d 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx @@ -147,6 +147,10 @@ export class StepIndexPattern extends Component { const { indexPatternCreationType } = this.props; const { existingIndexPatterns } = this.state; + const { http } = this.context.services; + const getIndexTags = (indexName: string) => indexPatternCreationType.getIndexTags(indexName); + const searchClient = this.context.services.data.search.search; + const showAllIndices = this.state.isIncludingSystemIndices; if ((existingIndexPatterns as string[]).includes(query)) { this.setState({ indexPatternExists: true }); @@ -157,12 +161,7 @@ export class StepIndexPattern extends Component indexPatternCreationType.getIndexTags(indexName), - query, - this.state.isIncludingSystemIndices - ) + getIndices({ http, getIndexTags, pattern: query, showAllIndices, searchClient }) ); // If the search changed, discard this state if (query !== this.lastQuery) { @@ -173,18 +172,8 @@ export class StepIndexPattern extends Component indexPatternCreationType.getIndexTags(indexName), - `${query}*`, - this.state.isIncludingSystemIndices - ), - getIndices( - this.context.services.http, - (indexName: string) => indexPatternCreationType.getIndexTags(indexName), - query, - this.state.isIncludingSystemIndices - ), + getIndices({ http, getIndexTags, pattern: `${query}*`, showAllIndices, searchClient }), + getIndices({ http, getIndexTags, pattern: query, showAllIndices, searchClient }), ]); // If the search changed, discard this state diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/create_index_pattern_wizard.tsx b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/create_index_pattern_wizard.tsx index aa97c21d766b9..259bd5c1d007e 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/create_index_pattern_wizard.tsx +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/create_index_pattern_wizard.tsx @@ -108,6 +108,11 @@ export class CreateIndexPatternWizard extends Component< }; fetchData = async () => { + const { http } = this.context.services; + const getIndexTags = (indexName: string) => + this.state.indexPatternCreationType.getIndexTags(indexName); + const searchClient = this.context.services.data.search.search; + const indicesFailMsg = ( this.state.indexPatternCreationType.getIndexTags(indexName), - `*`, - false - ), + getIndices({ http, getIndexTags, pattern: '*', searchClient }), [], indicesFailMsg @@ -142,12 +142,7 @@ export class CreateIndexPatternWizard extends Component< this.catchAndWarn( // if we get an error from remote cluster query, supply fallback value that allows user entry. // ['a'] is fallback value - getIndices( - this.context.services.http, - (indexName: string) => this.state.indexPatternCreationType.getIndexTags(indexName), - `*:*`, - false - ), + getIndices({ http, getIndexTags, pattern: '*:*', searchClient }), ['a'], clustersFailMsg diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.test.ts b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.test.ts index 44a2d1a3be0d0..f366085164631 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.test.ts +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.test.ts @@ -17,11 +17,12 @@ * under the License. */ -import { getIndices, responseToItemArray } from './get_indices'; +import { getIndices, responseToItemArray, dedupeMatchedItems } from './get_indices'; import { httpServiceMock } from '../../../../../../core/public/mocks'; -import { ResolveIndexResponseItemIndexAttrs } from '../types'; +import { ResolveIndexResponseItemIndexAttrs, MatchedItem } from '../types'; +import { Observable } from 'rxjs'; -export const successfulResponse = { +export const successfulResolveResponse = { indices: [ { name: 'remoteCluster1:bar-01', @@ -43,27 +44,64 @@ export const successfulResponse = { ], }; -const mockGetTags = () => []; +const successfulSearchResponse = { + rawResponse: { + aggregations: { + indices: { + buckets: [{ key: 'kibana_sample_data_ecommerce' }, { key: '.kibana_1' }], + }, + }, + }, +}; + +const getIndexTags = () => []; +const searchClient = () => + new Observable((observer) => { + observer.next(successfulSearchResponse); + observer.complete(); + }) as any; const http = httpServiceMock.createStartContract(); -http.get.mockResolvedValue(successfulResponse); +http.get.mockResolvedValue(successfulResolveResponse); describe('getIndices', () => { it('should work in a basic case', async () => { - const result = await getIndices(http, mockGetTags, 'kibana', false); + const uncalledSearchClient = jest.fn(); + const result = await getIndices({ + http, + getIndexTags, + pattern: 'kibana', + searchClient: uncalledSearchClient, + }); + expect(http.get).toHaveBeenCalled(); + expect(uncalledSearchClient).not.toHaveBeenCalled(); expect(result.length).toBe(3); expect(result[0].name).toBe('f-alias'); expect(result[1].name).toBe('foo'); }); + it('should make two calls in cross cluser case', async () => { + http.get.mockResolvedValue(successfulResolveResponse); + const result = await getIndices({ http, getIndexTags, pattern: '*:kibana', searchClient }); + + expect(http.get).toHaveBeenCalled(); + expect(result.length).toBe(4); + expect(result[0].name).toBe('f-alias'); + expect(result[1].name).toBe('foo'); + expect(result[2].name).toBe('kibana_sample_data_ecommerce'); + expect(result[3].name).toBe('remoteCluster1:bar-01'); + }); + it('should ignore ccs query-all', async () => { - expect((await getIndices(http, mockGetTags, '*:', false)).length).toBe(0); + expect((await getIndices({ http, getIndexTags, pattern: '*:', searchClient })).length).toBe(0); }); it('should ignore a single comma', async () => { - expect((await getIndices(http, mockGetTags, ',', false)).length).toBe(0); - expect((await getIndices(http, mockGetTags, ',*', false)).length).toBe(0); - expect((await getIndices(http, mockGetTags, ',foobar', false)).length).toBe(0); + expect((await getIndices({ http, getIndexTags, pattern: ',', searchClient })).length).toBe(0); + expect((await getIndices({ http, getIndexTags, pattern: ',*', searchClient })).length).toBe(0); + expect( + (await getIndices({ http, getIndexTags, pattern: ',foobar', searchClient })).length + ).toBe(0); }); it('response object to item array', () => { @@ -91,8 +129,14 @@ describe('getIndices', () => { }, ], }; - expect(responseToItemArray(result, mockGetTags)).toMatchSnapshot(); - expect(responseToItemArray({}, mockGetTags)).toEqual([]); + expect(responseToItemArray(result, getIndexTags)).toMatchSnapshot(); + expect(responseToItemArray({}, getIndexTags)).toEqual([]); + }); + + it('matched items are deduped', () => { + const setA = [{ name: 'a' }, { name: 'b' }] as MatchedItem[]; + const setB = [{ name: 'b' }, { name: 'c' }] as MatchedItem[]; + expect(dedupeMatchedItems(setA, setB)).toHaveLength(3); }); describe('errors', () => { @@ -100,7 +144,7 @@ describe('getIndices', () => { http.get.mockImplementationOnce(() => { throw new Error('Test error'); }); - const result = await getIndices(http, mockGetTags, 'kibana', false); + const result = await getIndices({ http, getIndexTags, pattern: 'kibana', searchClient }); expect(result.length).toBe(0); }); }); diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts index 6844e90316e22..618eb8e80a5ff 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts @@ -20,8 +20,11 @@ import { sortBy } from 'lodash'; import { HttpStart } from 'kibana/public'; import { i18n } from '@kbn/i18n'; +import { map, scan } from 'rxjs/operators'; import { IndexPatternCreationConfig } from '../../../../../index_pattern_management/public'; import { MatchedItem, ResolveIndexResponse, ResolveIndexResponseItemIndexAttrs } from '../types'; +import { DataPublicPluginStart, IEsSearchResponse } from '../../../../../data/public'; +import { MAX_SEARCH_SIZE } from '../constants'; const aliasLabel = i18n.translate('indexPatternManagement.aliasLabel', { defaultMessage: 'Alias' }); const dataStreamLabel = i18n.translate('indexPatternManagement.dataStreamLabel', { @@ -36,13 +39,134 @@ const frozenLabel = i18n.translate('indexPatternManagement.frozenLabel', { defaultMessage: 'Frozen', }); -export async function getIndices( - http: HttpStart, +export const searchResponseToArray = ( getIndexTags: IndexPatternCreationConfig['getIndexTags'], - rawPattern: string, showAllIndices: boolean -): Promise { +) => (response: IEsSearchResponse) => { + const { rawResponse } = response; + if (!rawResponse.aggregations) { + return []; + } else { + return rawResponse.aggregations.indices.buckets + .map((bucket: { key: string }) => { + return bucket.key; + }) + .filter((indexName: string) => { + if (showAllIndices) { + return true; + } else { + return !indexName.startsWith('.'); + } + }) + .map((indexName: string) => { + return { + name: indexName, + tags: getIndexTags(indexName), + item: {}, + }; + }); + } +}; + +export const getIndicesViaSearch = async ({ + getIndexTags, + pattern, + searchClient, + showAllIndices, +}: { + getIndexTags: IndexPatternCreationConfig['getIndexTags']; + pattern: string; + searchClient: DataPublicPluginStart['search']['search']; + showAllIndices: boolean; +}): Promise => + searchClient({ + params: { + ignoreUnavailable: true, + expand_wildcards: showAllIndices ? 'all' : 'open', + index: pattern, + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: MAX_SEARCH_SIZE, + }, + }, + }, + }, + }, + }) + .pipe(map(searchResponseToArray(getIndexTags, showAllIndices))) + .pipe(scan((accumulator = [], value) => accumulator.join(value))) + .toPromise() + .catch(() => []); + +export const getIndicesViaResolve = async ({ + http, + getIndexTags, + pattern, + showAllIndices, +}: { + http: HttpStart; + getIndexTags: IndexPatternCreationConfig['getIndexTags']; + pattern: string; + showAllIndices: boolean; +}) => + http + .get(`/internal/index-pattern-management/resolve_index/${pattern}`, { + query: showAllIndices ? { expand_wildcards: 'all' } : undefined, + }) + .then((response) => { + if (!response) { + return []; + } else { + return responseToItemArray(response, getIndexTags); + } + }); + +/** + * Takes two MatchedItem[]s and returns a merged set, with the second set prrioritized over the first based on name + * + * @param matchedA + * @param matchedB + */ + +export const dedupeMatchedItems = (matchedA: MatchedItem[], matchedB: MatchedItem[]) => { + const mergedMatchedItems = matchedA.reduce((col, item) => { + col[item.name] = item; + return col; + }, {} as Record); + + matchedB.reduce((col, item) => { + col[item.name] = item; + return col; + }, mergedMatchedItems); + + return Object.values(mergedMatchedItems).sort((a, b) => { + if (a.name > b.name) return 1; + if (b.name > a.name) return -1; + + return 0; + }); +}; + +export async function getIndices({ + http, + getIndexTags = () => [], + pattern: rawPattern, + showAllIndices = false, + searchClient, +}: { + http: HttpStart; + getIndexTags?: IndexPatternCreationConfig['getIndexTags']; + pattern: string; + showAllIndices?: boolean; + searchClient: DataPublicPluginStart['search']['search']; +}): Promise { const pattern = rawPattern.trim(); + const isCCS = pattern.indexOf(':') !== -1; + const requests: Array> = []; // Searching for `*:` fails for CCS environments. The search request // is worthless anyways as the we should only send a request @@ -62,20 +186,32 @@ export async function getIndices( return []; } - const query = showAllIndices ? { expand_wildcards: 'all' } : undefined; + const promiseResolve = getIndicesViaResolve({ + http, + getIndexTags, + pattern, + showAllIndices, + }).catch(() => []); + requests.push(promiseResolve); + + if (isCCS) { + // CCS supports ±1 major version. We won't be able to expect resolve endpoint to exist until v9 + const promiseSearch = getIndicesViaSearch({ + getIndexTags, + pattern, + searchClient, + showAllIndices, + }).catch(() => []); + requests.push(promiseSearch); + } - try { - const response = await http.get( - `/internal/index-pattern-management/resolve_index/${pattern}`, - { query } - ); - if (!response) { - return []; - } + const responses = await Promise.all(requests); - return responseToItemArray(response, getIndexTags); - } catch { - return []; + if (responses.length === 2) { + const [resolveResponse, searchResponse] = responses; + return dedupeMatchedItems(searchResponse, resolveResponse); + } else { + return responses[0]; } } diff --git a/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx b/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx index 2768314a75860..6d14f787135be 100644 --- a/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx +++ b/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx @@ -88,6 +88,7 @@ export const IndexPatternTable = ({ canSave, history }: Props) => { application, http, getMlCardState, + data, } = useKibana().services; const [indexPatterns, setIndexPatterns] = useState([]); const [creationOptions, setCreationOptions] = useState([]); @@ -122,24 +123,26 @@ export const IndexPatternTable = ({ canSave, history }: Props) => { const removeAliases = (item: MatchedItem) => !((item as unknown) as ResolveIndexResponseItemAlias).indices; + const searchClient = data.search.search; + const loadSources = () => { - getIndices(http, () => [], '*', false).then((dataSources) => + getIndices({ http, pattern: '*', searchClient }).then((dataSources) => setSources(dataSources.filter(removeAliases)) ); - getIndices(http, () => [], '*:*', false).then((dataSources) => + getIndices({ http, pattern: '*:*', searchClient }).then((dataSources) => setRemoteClustersExist(!!dataSources.filter(removeAliases).length) ); }; useEffect(() => { - getIndices(http, () => [], '*', false).then((dataSources) => { + getIndices({ http, pattern: '*', searchClient }).then((dataSources) => { setSources(dataSources.filter(removeAliases)); setIsLoadingSources(false); }); - getIndices(http, () => [], '*:*', false).then((dataSources) => + getIndices({ http, pattern: '*:*', searchClient }).then((dataSources) => setRemoteClustersExist(!!dataSources.filter(removeAliases).length) ); - }, [http, creationOptions]); + }, [http, creationOptions, searchClient]); chrome.docTitle.change(title);