diff --git a/docs/discover/kuery.asciidoc b/docs/discover/kuery.asciidoc index abf3e05fb7819..c835c15028074 100644 --- a/docs/discover/kuery.asciidoc +++ b/docs/discover/kuery.asciidoc @@ -19,14 +19,13 @@ they appear. This means documents with "quick brown fox" will match, but so will to search for a phrase. The query parser will no longer split on whitespace. Multiple search terms must be separated by explicit -boolean operators. Note that boolean operators are not case sensitive. +boolean operators. Lucene will combine search terms with an `or` by default, so `response:200 extension:php` would +become `response:200 or extension:php` in KQL. This will match documents where response matches 200, extension matches php, or both. +Note that boolean operators are not case sensitive. -`response:200 extension:php` in lucene would become `response:200 and extension:php`. - This will match documents where response matches 200 and extension matches php. +We can make terms required by using `and`. -We can make terms optional by using `or`. - -`response:200 or extension:php` will match documents where response matches 200, extension matches php, or both. +`response:200 and extension:php` will match documents where response matches 200 and extension matches php. By default, `and` has a higher precedence than `or`. @@ -73,7 +72,7 @@ set these terms will be matched against all fields. For example, a query for `re in the response field, but a query for just `200` will search for 200 across all fields in your index. ============ -===== Nested Field Support +==== Nested Field Support KQL supports querying on {ref}/nested.html[nested fields] through a special syntax. You can query nested fields in subtly different ways, depending on the results you want, so crafting nested queries requires extra thought. @@ -85,7 +84,8 @@ There are two main approaches to take: * *Parts of the query can match different nested documents.* This is how a regular object field works. Although generally less useful, there might be occasions where you want to query a nested field in this way. -Let's take a look at the first approach. In the following document, `items` is a nested field: +Let's take a look at the first approach. In the following document, `items` is a nested field. Each document in the nested +field contains a name, stock, and category. [source,json] ---------------------------------- @@ -116,21 +116,38 @@ Let's take a look at the first approach. In the following document, `items` is a } ---------------------------------- +===== Match a single nested document + To find stores that have more than 10 bananas in stock, you would write a query like this: `items:{ name:banana and stock > 10 }` -`items` is the "nested path". Everything inside the curly braces (the "nested group") must match a single document. -For example, `items:{ name:banana and stock:9 }` does not match because there isn't a single nested document that -matches the entire query in the nested group. +`items` is the "nested path". Everything inside the curly braces (the "nested group") must match a single nested document. + +The following example returns no matches because no single nested document has bananas with a stock of 9. + +`items:{ name:banana and stock:9 }` + +==== Match different nested documents -What if you want to find a store with more than 10 bananas that *also* stocks vegetables? This is the second way of querying a nested field, and you can do it like this: +The subqueries in this example are in separate nested groups and can match different nested documents. + +`items:{ name:banana } and items:{ stock:9 }` + +`name:banana` matches the first document in the array and `stock:9` matches the third document in the array. + +==== Combine approaches + +You can combine these two approaches to create complex queries. What if you wanted to find a store with more than 10 +bananas that *also* stocks vegetables? You could do this: `items:{ name:banana and stock > 10 } and items:{ category:vegetable }` The first nested group (`name:banana and stock > 10`) must still match a single document, but the `category:vegetables` subquery can match a different nested document because it is in a separate group. +==== Nested fields inside other nested fields + KQL's syntax also supports nested fields inside of other nested fields—you simply have to specify the full path. Suppose you have a document where `level1` and `level2` are both nested fields: diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index 80a6a96aeaf2b..4c4f321695d70 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -405,5 +405,29 @@ describe('Filter Utils', () => { }, ]); }); + + test('Return Error if filter is using an non-existing key null key', () => { + const validationObject = validateFilterKueryNode( + fromKueryExpression('foo.attributes.description: hello AND bye'), + ['foo'], + mockMappings + ); + expect(validationObject).toEqual([ + { + astPath: 'arguments.0', + error: null, + isSavedObjectAttr: false, + key: 'foo.attributes.description', + type: 'foo', + }, + { + astPath: 'arguments.1', + error: 'The key is empty and needs to be wrapped by a saved object type like foo', + isSavedObjectAttr: false, + key: null, + type: null, + }, + ]); + }); }); }); diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index 64abf268cacd6..e331d3eff990f 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -128,7 +128,8 @@ export const validateFilterKueryNode = ( }, []); }; -const getType = (key: string) => (key.includes('.') ? key.split('.')[0] : null); +const getType = (key: string | undefined | null) => + key != null && key.includes('.') ? key.split('.')[0] : null; /** * Is this filter key referring to a a top-level SavedObject attribute such as @@ -137,8 +138,8 @@ const getType = (key: string) => (key.includes('.') ? key.split('.')[0] : null); * @param key * @param indexMapping */ -export const isSavedObjectAttr = (key: string, indexMapping: IndexMapping) => { - const keySplit = key.split('.'); +export const isSavedObjectAttr = (key: string | null | undefined, indexMapping: IndexMapping) => { + const keySplit = key != null ? key.split('.') : []; if (keySplit.length === 1 && fieldDefined(indexMapping, keySplit[0])) { return true; } else if (keySplit.length === 2 && fieldDefined(indexMapping, keySplit[1])) { @@ -149,10 +150,13 @@ export const isSavedObjectAttr = (key: string, indexMapping: IndexMapping) => { }; export const hasFilterKeyError = ( - key: string, + key: string | null | undefined, types: string[], indexMapping: IndexMapping ): string | null => { + if (key == null) { + return `The key is empty and needs to be wrapped by a saved object type like ${types.join()}`; + } if (!key.includes('.')) { return `This key '${key}' need to be wrapped by a saved object type like ${types.join()}`; } else if (key.includes('.')) { diff --git a/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.test.ts b/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.test.ts index a6ca444de6d4c..af142973a535d 100644 --- a/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.test.ts +++ b/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.test.ts @@ -25,7 +25,7 @@ describe('getNotifyUserAboutOptInDefault: get a flag that describes if the user getNotifyUserAboutOptInDefault({ allowChangingOptInStatus: true, telemetrySavedObject: { userHasSeenNotice: false }, - telemetryOptedIn: null, + telemetryOptedIn: true, configTelemetryOptIn: true, }) ).toBe(true); @@ -40,50 +40,37 @@ describe('getNotifyUserAboutOptInDefault: get a flag that describes if the user configTelemetryOptIn: false, }) ).toBe(false); - }); - it('should return false if user has seen notice', () => { expect( getNotifyUserAboutOptInDefault({ - allowChangingOptInStatus: true, - telemetrySavedObject: { userHasSeenNotice: true }, - telemetryOptedIn: false, + allowChangingOptInStatus: false, + telemetrySavedObject: null, + telemetryOptedIn: true, configTelemetryOptIn: true, }) ).toBe(false); + }); + it('should return false if user has seen notice', () => { expect( getNotifyUserAboutOptInDefault({ allowChangingOptInStatus: true, telemetrySavedObject: { userHasSeenNotice: true }, - telemetryOptedIn: true, - configTelemetryOptIn: true, + telemetryOptedIn: false, + configTelemetryOptIn: false, }) ).toBe(false); - }); - it('not show notice for users already opted in and has not seen notice yet', () => { expect( getNotifyUserAboutOptInDefault({ allowChangingOptInStatus: true, - telemetrySavedObject: { userHasSeenNotice: false }, + telemetrySavedObject: { userHasSeenNotice: true }, telemetryOptedIn: true, configTelemetryOptIn: true, }) ).toBe(false); }); - it('should see notice if they are merely opted in by default and have not yet seen the notice', () => { - expect( - getNotifyUserAboutOptInDefault({ - allowChangingOptInStatus: true, - telemetrySavedObject: { userHasSeenNotice: false }, - telemetryOptedIn: null, - configTelemetryOptIn: true, - }) - ).toBe(true); - }); - it('should return false if user is opted out', () => { expect( getNotifyUserAboutOptInDefault({ diff --git a/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.ts b/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.ts index eb95aff6392e0..8ef3bd8388ecb 100644 --- a/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.ts +++ b/src/legacy/core_plugins/telemetry/server/telemetry_config/get_telemetry_notify_user_about_optin_default.ts @@ -41,9 +41,5 @@ export function getNotifyUserAboutOptInDefault({ return false; } - if (telemetryOptedIn !== null) { - return false; // they were not defaulted in - } - - return configTelemetryOptIn; + return telemetryOptedIn === true && configTelemetryOptIn === true; } diff --git a/src/plugins/data/common/es_query/filters/phrase_filter.ts b/src/plugins/data/common/es_query/filters/phrase_filter.ts index 8b8c5f8915269..167bb3f840350 100644 --- a/src/plugins/data/common/es_query/filters/phrase_filter.ts +++ b/src/plugins/data/common/es_query/filters/phrase_filter.ts @@ -25,6 +25,12 @@ export type PhraseFilterMeta = FilterMeta & { params?: { query: string; // The unformatted value }; + field?: any; + index?: any; +}; + +export type PhraseFilter = Filter & { + meta: PhraseFilterMeta; script?: { script: { source?: any; @@ -32,12 +38,6 @@ export type PhraseFilterMeta = FilterMeta & { params: any; }; }; - field?: any; - index?: any; -}; - -export type PhraseFilter = Filter & { - meta: PhraseFilterMeta; }; type PhraseFilterValue = string | number | boolean; @@ -79,7 +79,7 @@ export const buildPhraseFilter = ( return { meta: { index: indexPattern.id, field: field.name } as PhraseFilterMeta, script: getPhraseScript(field, value), - } as PhraseFilter; + }; } else { return { meta: { index: indexPattern.id }, diff --git a/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts b/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts index 42607843df3ba..f2fd55af4f418 100644 --- a/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts +++ b/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts @@ -42,7 +42,7 @@ function getExistingFilter( } if (esFilters.isScriptedPhraseFilter(filter)) { - return filter.meta.field === fieldName && filter.meta.script!.script.params.value === value; + return filter.meta.field === fieldName && filter.script!.script.params.value === value; } }); } diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.ts index fbdf496ebaec4..c260a754e4594 100644 --- a/x-pack/legacy/plugins/alerting/server/alerts_client.ts +++ b/x-pack/legacy/plugins/alerting/server/alerts_client.ts @@ -32,7 +32,7 @@ interface ConstructorOptions { createAPIKey: () => Promise; } -interface FindOptions { +export interface FindOptions { options?: { perPage?: number; page?: number; @@ -40,6 +40,7 @@ interface FindOptions { defaultSearchOperator?: 'AND' | 'OR'; searchFields?: string[]; sortField?: string; + sortOrder?: string; hasReference?: { type: string; id: string; diff --git a/x-pack/legacy/plugins/canvas/shareable_runtime/components/__tests__/app.test.tsx b/x-pack/legacy/plugins/canvas/shareable_runtime/components/__tests__/app.test.tsx index 9cf2ddc3a22e3..2ec3cfde8bd68 100644 --- a/x-pack/legacy/plugins/canvas/shareable_runtime/components/__tests__/app.test.tsx +++ b/x-pack/legacy/plugins/canvas/shareable_runtime/components/__tests__/app.test.tsx @@ -19,8 +19,6 @@ import { getScrubber as scrubber, getScrubberSlideContainer as scrubberContainer, getPageControlsCenter as center, - getSettingsTrigger as trigger, - getContextMenuItems as menuItems, // getAutoplayTextField as autoplayText, // getAutoplayCheckbox as autoplayCheck, // getAutoplaySubmit as autoplaySubmit, @@ -30,6 +28,7 @@ import { getPageControlsPrevious as previous, getPageControlsNext as next, } from '../../test/selectors'; +import { openSettings, selectMenuItem } from '../../test/interactions'; // Mock the renderers jest.mock('../../supported_renderers'); @@ -102,13 +101,9 @@ describe('', () => { test('autohide footer functions on mouseEnter + Leave', async () => { const wrapper = getWrapper(); - trigger(wrapper).simulate('click'); - await tick(20); - menuItems(wrapper) - .at(1) - .simulate('click'); - await tick(20); - wrapper.update(); + await openSettings(wrapper); + await selectMenuItem(wrapper, 1); + expect(footer(wrapper).prop('isHidden')).toEqual(false); expect(footer(wrapper).prop('isAutohide')).toEqual(false); toolbarCheck(wrapper).simulate('click'); @@ -125,13 +120,9 @@ describe('', () => { expect(scrubber(wrapper).prop('isScrubberVisible')).toEqual(true); // Open the menu and activate toolbar hiding. - trigger(wrapper).simulate('click'); - await tick(20); - menuItems(wrapper) - .at(1) - .simulate('click'); - await tick(20); - wrapper.update(); + await openSettings(wrapper); + await selectMenuItem(wrapper, 1); + toolbarCheck(wrapper).simulate('click'); await tick(20); diff --git a/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/__snapshots__/settings.test.tsx.snap b/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/__snapshots__/settings.test.tsx.snap index 31d89a4e68c4f..fbd8c9c5ab0c3 100644 --- a/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/__snapshots__/settings.test.tsx.snap +++ b/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/__snapshots__/settings.test.tsx.snap @@ -56,26 +56,35 @@ exports[` can navigate Autoplay Settings 1`] = ` class="euiContextMenu__itemLayout" > + > + + Auto Play + > + + @@ -498,26 +516,35 @@ exports[` can navigate Toolbar Settings, closes when activated 1`] = class="euiContextMenu__itemLayout" > + > + + Auto Play + > + + @@ -566,6 +602,258 @@ exports[` can navigate Toolbar Settings, closes when activated 1`] = `; -exports[` can navigate Toolbar Settings, closes when activated 2`] = `"
Settings

Hide Toolbar

Hide the toolbar when the mouse is not within the Canvas?
"`; +exports[` can navigate Toolbar Settings, closes when activated 2`] = ` +
+
+
+
+
+ +
+
+
+`; exports[` can navigate Toolbar Settings, closes when activated 3`] = `"
Settings

Hide Toolbar

Hide the toolbar when the mouse is not within the Canvas?
"`; diff --git a/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/settings.test.tsx b/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/settings.test.tsx index 0667674b6a7dd..66515eb3421d5 100644 --- a/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/settings.test.tsx +++ b/x-pack/legacy/plugins/canvas/shareable_runtime/components/footer/settings/__tests__/settings.test.tsx @@ -7,7 +7,8 @@ import { mount, ReactWrapper } from 'enzyme'; import React from 'react'; import { JestContext } from '../../../../test/context_jest'; -import { takeMountedSnapshot, tick } from '../../../../test'; +import { takeMountedSnapshot } from '../../../../test'; +import { openSettings, selectMenuItem } from '../../../../test/interactions'; import { getSettingsTrigger as trigger, getPopover as popover, @@ -60,36 +61,26 @@ describe('', () => { expect(popover(wrapper).prop('isOpen')).toEqual(false); }); - test.skip('can navigate Autoplay Settings', async () => { - trigger(wrapper).simulate('click'); + test('can navigate Autoplay Settings', async () => { + await openSettings(wrapper); expect(takeMountedSnapshot(portal(wrapper))).toMatchSnapshot(); - await tick(20); - menuItems(wrapper) - .at(0) - .simulate('click'); - await tick(20); + + await selectMenuItem(wrapper, 0); expect(takeMountedSnapshot(portal(wrapper))).toMatchSnapshot(); }); - test.skip('can navigate Toolbar Settings, closes when activated', async () => { - trigger(wrapper).simulate('click'); + test('can navigate Toolbar Settings, closes when activated', async () => { + await openSettings(wrapper); expect(takeMountedSnapshot(portal(wrapper))).toMatchSnapshot(); - menuItems(wrapper) - .at(1) - .simulate('click'); - // Wait for the animation and DOM update - await tick(40); - portal(wrapper).update(); - expect(portal(wrapper).html()).toMatchSnapshot(); + await selectMenuItem(wrapper, 1); + expect(takeMountedSnapshot(portal(wrapper))).toMatchSnapshot(); // Click the Hide Toolbar switch portal(wrapper) .find('button[data-test-subj="hideToolbarSwitch"]') .simulate('click'); - // Wait for the animation and DOM update - await tick(20); portal(wrapper).update(); // The Portal should not be open. diff --git a/x-pack/legacy/plugins/canvas/shareable_runtime/test/interactions.ts b/x-pack/legacy/plugins/canvas/shareable_runtime/test/interactions.ts new file mode 100644 index 0000000000000..1c5b78929aaa5 --- /dev/null +++ b/x-pack/legacy/plugins/canvas/shareable_runtime/test/interactions.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ReactWrapper } from 'enzyme'; +import { getSettingsTrigger, getPortal, getContextMenuItems } from './selectors'; +import { waitFor } from './utils'; + +export const openSettings = async function(wrapper: ReactWrapper) { + getSettingsTrigger(wrapper).simulate('click'); + + try { + // Wait for EuiPanel to be visible + await waitFor(() => { + wrapper.update(); + + return getPortal(wrapper) + .find('EuiPanel') + .exists(); + }); + } catch (e) { + throw new Error('Settings Panel did not open in given time'); + } +}; + +export const selectMenuItem = async function(wrapper: ReactWrapper, menuItemIndex: number) { + getContextMenuItems(wrapper) + .at(menuItemIndex) + .simulate('click'); + + try { + // When the menu item is clicked, wait for all of the context menus to be there + await waitFor(() => { + wrapper.update(); + return getPortal(wrapper).find('EuiContextMenuPanel').length === 2; + }); + } catch (e) { + throw new Error('Context menu did not transition'); + } +}; diff --git a/x-pack/legacy/plugins/canvas/shareable_runtime/test/utils.ts b/x-pack/legacy/plugins/canvas/shareable_runtime/test/utils.ts index 2e7bc4b262b52..4e18f2af1b06a 100644 --- a/x-pack/legacy/plugins/canvas/shareable_runtime/test/utils.ts +++ b/x-pack/legacy/plugins/canvas/shareable_runtime/test/utils.ts @@ -6,7 +6,6 @@ import { ReactWrapper } from 'enzyme'; import { Component } from 'react'; -import { setTimeout } from 'timers'; export const tick = (ms = 0) => new Promise(resolve => { @@ -19,3 +18,25 @@ export const takeMountedSnapshot = (mountedComponent: ReactWrapper<{}, {}, Compo template.innerHTML = html; return template.content.firstChild; }; + +export const waitFor = (fn: () => boolean, stepMs = 100, failAfterMs = 1000) => { + return new Promise((resolve, reject) => { + let waitForTimeout: NodeJS.Timeout; + + const tryCondition = () => { + if (fn()) { + clearTimeout(failTimeout); + resolve(); + } else { + waitForTimeout = setTimeout(tryCondition, stepMs); + } + }; + + const failTimeout = setTimeout(() => { + clearTimeout(waitForTimeout); + reject('wait for condition was never met'); + }, failAfterMs); + + tryCondition(); + }); +}; diff --git a/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.test.tsx b/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.test.tsx index 1bd30bad818d1..988bb13841fa5 100644 --- a/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.test.tsx +++ b/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.test.tsx @@ -23,10 +23,7 @@ describe('Scroll to top', () => { Object.defineProperty(globalNode.window, 'scroll', { value: spyScroll }); mount( useScrollToTop()} />); - expect(spyScroll).toHaveBeenCalledWith({ - top: 0, - left: 0, - }); + expect(spyScroll).toHaveBeenCalledWith(0, 0); }); test('scrollTo have been called', () => { diff --git a/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.tsx b/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.tsx index 59f9c99d6ab8e..8d4548516fc16 100644 --- a/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.tsx +++ b/x-pack/legacy/plugins/siem/public/components/scroll_to_top/index.tsx @@ -10,10 +10,7 @@ export const useScrollToTop = () => { useEffect(() => { // trying to use new API - https://developer.mozilla.org/en-US/docs/Web/API/Window/scrollTo if (window.scroll) { - window.scroll({ - top: 0, - left: 0, - }); + window.scroll(0, 0); } else { // just a fallback for older browsers window.scrollTo(0, 0); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.test.ts new file mode 100644 index 0000000000000..7873781fb05c4 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.test.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getFilter } from './find_signals'; +import { SIGNALS_ID } from '../../../../common/constants'; + +describe('find_signals', () => { + test('it returns a full filter with an AND if sent down', () => { + expect(getFilter('alert.attributes.enabled: true')).toEqual( + `alert.attributes.alertTypeId: ${SIGNALS_ID} AND alert.attributes.enabled: true` + ); + }); + + test('it returns existing filter with no AND when not set', () => { + expect(getFilter(null)).toEqual(`alert.attributes.alertTypeId: ${SIGNALS_ID}`); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts index 23f4e38a95eea..63e6a069c0cfe 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/find_signals.ts @@ -7,12 +7,31 @@ import { SIGNALS_ID } from '../../../../common/constants'; import { FindSignalParams } from './types'; -export const findSignals = async ({ alertsClient, perPage, page, fields }: FindSignalParams) => - alertsClient.find({ +export const getFilter = (filter: string | null | undefined) => { + if (filter == null) { + return `alert.attributes.alertTypeId: ${SIGNALS_ID}`; + } else { + return `alert.attributes.alertTypeId: ${SIGNALS_ID} AND ${filter}`; + } +}; + +export const findSignals = async ({ + alertsClient, + perPage, + page, + fields, + filter, + sortField, + sortOrder, +}: FindSignalParams) => { + return alertsClient.find({ options: { fields, page, perPage, - filter: `alert.attributes.alertTypeId: ${SIGNALS_ID}`, + filter: getFilter(filter), + sortOrder, + sortField, }, }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index a6cb56ada8df1..723e2aad7fe6a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -70,7 +70,9 @@ export interface FindParamsRest { per_page: number; page: number; sort_field: string; + sort_order: 'asc' | 'desc'; fields: string[]; + filter: string; } export interface Clients { @@ -95,7 +97,9 @@ export interface FindSignalsRequest extends Omit { page: number; search?: string; sort_field?: string; + filter?: string; fields?: string[]; + sort_order?: 'asc' | 'desc'; }; } @@ -104,7 +108,9 @@ export interface FindSignalParams { perPage?: number; page?: number; sortField?: string; + filter?: string; fields?: string[]; + sortOrder?: 'asc' | 'desc'; } export interface ReadSignalParams { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index a5e6d03a3378b..d8ba455445c0c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -293,6 +293,28 @@ describe('utils', () => { ); expect(result).toEqual(true); }); + test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => { + const sampleParams = sampleSignalAlertParams(10); + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(sampleEmptyDocSearchResults); + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger, + sampleSignalId + ); + expect(result).toEqual(true); + }); test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { const sampleParams = sampleSignalAlertParams(5); mockService.callCluster diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index 2967f41ffb697..80530f9c2245f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -157,7 +157,9 @@ export const searchAfterAndBulkIndex = async ( service, logger ); - sortIds = searchAfterResult.hits.hits[0].sort; + if (searchAfterResult.hits.hits.length === 0) { + return true; + } hitsSize += searchAfterResult.hits.hits.length; logger.debug(`size adjusted: ${hitsSize}`); sortIds = searchAfterResult.hits.hits[0].sort; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts index 18252c4f27fb0..120b71fab7d3a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/find_signals_route.ts @@ -39,6 +39,8 @@ export const createFindSignalRoute: Hapi.ServerRoute = { perPage: query.per_page, page: query.page, sortField: query.sort_field, + sortOrder: query.sort_order, + filter: query.filter, }); return transformFindAlertsOrError(signals); }, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts index ecb42399932f6..352d8d57cdb83 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts @@ -1446,6 +1446,8 @@ describe('schemas', () => { page: 1, sort_field: 'some field', fields: ['field 1', 'field 2'], + filter: 'some filter', + sort_order: 'asc', }).error ).toBeFalsy(); }); @@ -1505,6 +1507,68 @@ describe('schemas', () => { test('page has a default of 1', () => { expect(findSignalsSchema.validate>({}).value.page).toEqual(1); }); + + test('filter works with a string', () => { + expect( + findSignalsSchema.validate>({ + filter: 'some value 1', + }).error + ).toBeFalsy(); + }); + + test('filter does not work with a number', () => { + expect( + findSignalsSchema.validate> & { filter: number }>({ + filter: 5, + }).error + ).toBeTruthy(); + }); + + test('sort_order requires sort_field to work', () => { + expect( + findSignalsSchema.validate>({ + sort_order: 'asc', + }).error + ).toBeTruthy(); + }); + + // TODO: Delete this if not used + test.skip('sort_field requires sort_order to work', () => { + expect( + findSignalsSchema.validate>({ + sort_field: 'some field', + }).error + ).toBeTruthy(); + }); + + test('sort_order and sort_field validate together', () => { + expect( + findSignalsSchema.validate>({ + sort_order: 'asc', + sort_field: 'some field', + }).error + ).toBeFalsy(); + }); + + test('sort_order validates with desc and sort_field', () => { + expect( + findSignalsSchema.validate>({ + sort_order: 'desc', + sort_field: 'some field', + }).error + ).toBeFalsy(); + }); + + test('sort_order does not validate with a string other than asc and desc', () => { + expect( + findSignalsSchema.validate< + Partial> & { sort_order: string } + >({ + sort_order: 'some other string', + sort_field: 'some field', + }).error + ).toBeTruthy(); + }); }); describe('querySignalSchema', () => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts index 596850b4a11e4..446fa7cb305b9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts @@ -28,6 +28,7 @@ const name = Joi.string(); const severity = Joi.string(); const to = Joi.string(); const type = Joi.string().valid('filter', 'query', 'saved_query'); +const queryFilter = Joi.string(); const references = Joi.array() .items(Joi.string()) .single(); @@ -38,6 +39,7 @@ const page = Joi.number() .min(1) .default(1); const sort_field = Joi.string(); +const sort_order = Joi.string().valid('asc', 'desc'); const tags = Joi.array().items(Joi.string()); const fields = Joi.array() .items(Joi.string()) @@ -113,8 +115,14 @@ export const querySignalSchema = Joi.object({ }).xor('id', 'rule_id'); export const findSignalsSchema = Joi.object({ + fields, + filter: queryFilter, per_page, page, - sort_field, - fields, + sort_field: Joi.when(Joi.ref('sort_order'), { + is: Joi.exist(), + then: sort_field.required(), + otherwise: sort_field.optional(), + }), + sort_order, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts index 69f25e84d995c..fc6aefc72e33d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts @@ -25,6 +25,7 @@ describe('utils', () => { false_positives: [], from: 'now-6m', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -51,6 +52,7 @@ describe('utils', () => { enabled: true, false_positives: [], id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -78,6 +80,7 @@ describe('utils', () => { false_positives: [], from: 'now-6m', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -105,6 +108,7 @@ describe('utils', () => { false_positives: [], from: 'now-6m', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -131,6 +135,7 @@ describe('utils', () => { description: 'Detecting root and admin users', false_positives: [], id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -145,6 +150,64 @@ describe('utils', () => { type: 'query', }); }); + + test('should return enabled is equal to false', () => { + const fullSignal = getResult(); + fullSignal.enabled = false; + const signalWithEnabledFalse = transformAlertToSignal(fullSignal); + expect(signalWithEnabledFalse).toEqual({ + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + from: 'now-6m', + false_positives: [], + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + rule_id: 'rule-1', + max_signals: 100, + name: 'Detect Root/Admin Users', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + size: 1, + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }); + }); + + test('should return immutable is equal to false', () => { + const fullSignal = getResult(); + fullSignal.alertTypeParams.immutable = false; + const signalWithEnabledFalse = transformAlertToSignal(fullSignal); + expect(signalWithEnabledFalse).toEqual({ + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + from: 'now-6m', + false_positives: [], + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + rule_id: 'rule-1', + max_signals: 100, + name: 'Detect Root/Admin Users', + query: 'user.name: root or user.name: admin', + references: ['http://www.example.com', 'https://ww.example.com'], + severity: 'high', + size: 1, + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + }); + }); }); describe('getIdError', () => { @@ -208,6 +271,7 @@ describe('utils', () => { false_positives: [], from: 'now-6m', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', @@ -243,6 +307,7 @@ describe('utils', () => { false_positives: [], from: 'now-6m', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', rule_id: 'rule-1', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts index 4d653210b2bff..fac30abd6992d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts @@ -5,7 +5,7 @@ */ import Boom from 'boom'; -import { pickBy, identity } from 'lodash/fp'; +import { pickBy } from 'lodash/fp'; import { SignalAlertType, isAlertType, OutputSignalAlertRest, isAlertTypes } from '../alerts/types'; export const getIdError = ({ @@ -27,7 +27,7 @@ export const getIdError = ({ // Transforms the data but will remove any null or undefined it encounters and not include // those on the export export const transformAlertToSignal = (signal: SignalAlertType): Partial => { - return pickBy(identity, { + return pickBy((value: unknown) => value != null, { created_by: signal.createdBy, description: signal.alertTypeParams.description, enabled: signal.enabled, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signal_by_filter.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signal_by_filter.sh new file mode 100755 index 0000000000000..6136f66025f3d --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signal_by_filter.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +FILTER=${1:-'alert.attributes.enabled:%20true'} + +# Example: ./find_signal_by_filter.sh "alert.attributes.enabled:%20true" +# Example: ./find_signal_by_filter.sh "alert.attributes.name:%20Detect*" +# The %20 is just an encoded space that is typical of URL's. +# Table of them for testing if needed: https://www.w3schools.com/tags/ref_urlencode.asp +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}/api/detection_engine/rules/_find?filter=$FILTER | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals_sort.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals_sort.sh new file mode 100755 index 0000000000000..3f8bab28544e3 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_signals_sort.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +SORT=${1:-'enabled'} +ORDER=${2:-'asc'} + +# Example: ./find_signals_sort.sh enabled asc +curl -s -k \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET "${KIBANA_URL}/api/detection_engine/rules/_find?sort_field=$SORT&sort_order=$ORDER" \ + | jq .