diff --git a/docs/user/dashboard/lens.asciidoc b/docs/user/dashboard/lens.asciidoc index 8623b8c3ca107..3c2a120d167d9 100644 --- a/docs/user/dashboard/lens.asciidoc +++ b/docs/user/dashboard/lens.asciidoc @@ -96,13 +96,37 @@ All columns that belong to the same layer pane group are sorted in the table. * *Text alignment* — Aligns the values in the cell to the *Left*, *Center*, or *Right*. +* *Color by value* — Applies color to the cell or text values. To change the color, click the *Edit colors* icon. + * *Hide column* — Hides the column for the field. * *Directly filter on click* — Turns column values into clickable links that allow you to filter or drill down into the data. * *Summary row* — Adds a row that displays the summary value. When specified, allows you to enter a *Summary label*. -* *Color by value* — Applies color to the cell or text values. To change the color, click *Edit*. +[float] +[[assign-colors-to-terms]] +===== Assign colors to terms + +preview::[] + +For term-based metrics, assign a color to each term with color mapping. + +. Create a custom table. + +. In the layer pane, select a *Rows* or *Metrics* field. + +. In the *Color by value* option, select *Cell* or *Text*. + +. Click the *Edit colors* icon. + +. Toggle the button to use the Color Mapping feature. + +. Select a color palette and mode. + +. Click *Add assignment* to assign a color to a specific term, or click *Add all unassigned terms* to assign colors to all terms. Assigning colors to dates is unsupported. + +. Configure color assignments. You can also select whether unassigned terms should be mapped to the selected color palette or a single color. [float] [[drag-and-drop-keyboard-navigation]] diff --git a/package.json b/package.json index afda7cd4c9125..0ad1d96d7b5db 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,7 @@ "@elastic/datemath": "5.0.3", "@elastic/ebt": "^1.1.1", "@elastic/ecs": "^8.11.1", - "@elastic/elasticsearch": "^8.15.0", + "@elastic/elasticsearch": "^8.15.1", "@elastic/ems-client": "8.5.3", "@elastic/eui": "97.3.0", "@elastic/filesaver": "1.1.2", @@ -1498,7 +1498,7 @@ "@octokit/rest": "^17.11.2", "@parcel/watcher": "^2.1.0", "@playwright/test": "=1.46.0", - "@redocly/cli": "^1.25.10", + "@redocly/cli": "^1.25.11", "@statoscope/webpack-plugin": "^5.28.2", "@storybook/addon-a11y": "^6.5.16", "@storybook/addon-actions": "^6.5.16", diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts index 1aeabb7e86dea..c84b30cf15774 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/catch_retryable_es_client_errors.test.ts @@ -72,24 +72,25 @@ describe('catchRetryableEsClientErrors', () => { type: 'retryable_es_client_error', }); }); - it('ResponseError with retryable status code', async () => { - const statusCodes = [503, 401, 403, 408, 410, 429]; - return Promise.all( - statusCodes.map(async (status) => { - const error = new esErrors.ResponseError( - elasticsearchClientMock.createApiResponse({ - statusCode: status, - body: { error: { type: 'reason' } }, - }) - ); - expect( - ((await Promise.reject(error).catch(catchRetryableEsClientErrors)) as any).left - ).toMatchObject({ - message: 'reason', - type: 'retryable_es_client_error', - }); - }) - ); - }); + it.each([503, 401, 403, 408, 410, 429])( + 'ResponseError with retryable status code (%d)', + async (status) => { + const error = new esErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: status, + body: { error: { type: 'reason' } }, + }) + ); + expect( + ((await Promise.reject(error).catch(catchRetryableEsClientErrors)) as any).left + ).toMatchObject({ + message: + status === 410 + ? 'This API is unavailable in the version of Elasticsearch you are using.' + : 'reason', + type: 'retryable_es_client_error', + }); + } + ); }); }); diff --git a/packages/kbn-apm-synthtrace/src/lib/shared/base_client.ts b/packages/kbn-apm-synthtrace/src/lib/shared/base_client.ts index ed6d1b813184b..7fd639331af80 100644 --- a/packages/kbn-apm-synthtrace/src/lib/shared/base_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/shared/base_client.ts @@ -55,7 +55,6 @@ export class SynthtraceEsClient { await this.client.indices.resolveIndex({ name: this.indices.join(','), expand_wildcards: ['open', 'hidden'], - // @ts-expect-error ignore_unavailable is not in the type definition, but it is accepted by es ignore_unavailable: true, }) ).indices.map((index: { name: string }) => index.name) diff --git a/src/plugins/controls/common/constants.ts b/src/plugins/controls/common/constants.ts index d1434d4df2ae0..afd6fe66f0df1 100644 --- a/src/plugins/controls/common/constants.ts +++ b/src/plugins/controls/common/constants.ts @@ -16,7 +16,7 @@ export const CONTROL_CHAINING_OPTIONS = { NONE: 'NONE', HIERARCHICAL: 'HIERARCHI export const DEFAULT_CONTROL_WIDTH: ControlWidth = CONTROL_WIDTH_OPTIONS.MEDIUM; export const DEFAULT_CONTROL_LABEL_POSITION: ControlLabelPosition = CONTROL_LABEL_POSITION_OPTIONS.ONE_LINE; -export const DEFAULT_CONTROL_GROW: boolean = true; +export const DEFAULT_CONTROL_GROW: boolean = false; export const DEFAULT_CONTROL_CHAINING: ControlGroupChainingSystem = CONTROL_CHAINING_OPTIONS.HIERARCHICAL; export const DEFAULT_IGNORE_PARENT_SETTINGS = { diff --git a/src/plugins/controls/public/control_group/components/control_group_editor.tsx b/src/plugins/controls/public/control_group/components/control_group_editor.tsx index 8f1ccb4d699b0..cb21c23bc9ce4 100644 --- a/src/plugins/controls/public/control_group/components/control_group_editor.tsx +++ b/src/plugins/controls/public/control_group/components/control_group_editor.tsx @@ -72,7 +72,7 @@ export const ControlGroupEditor = ({ onCancel, onSave, onDeleteAll, stateManager return ( <> - +

{ControlGroupStrings.management.getFlyoutTitle()}

 @@ -80,7 +80,7 @@ export const ControlGroupEditor = ({ onCancel, onSave, onDeleteAll, stateManager { onCancel(); }} @@ -204,7 +204,7 @@ export const ControlGroupEditor = ({ onCancel, onSave, onDeleteAll, stateManager { diff --git a/src/plugins/controls/public/control_group/control_group_strings.tsx b/src/plugins/controls/public/control_group/control_group_strings.tsx index b8f6a11abf839..f5c92d987b271 100644 --- a/src/plugins/controls/public/control_group/control_group_strings.tsx +++ b/src/plugins/controls/public/control_group/control_group_strings.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; export const ControlGroupStrings = { getSaveChangesTitle: () => i18n.translate('controls.controlGroup.manageControl.saveChangesTitle', { - defaultMessage: 'Save and close', + defaultMessage: 'Save', }), getCancelTitle: () => i18n.translate('controls.controlGroup.manageControl.cancelTitle', { diff --git a/src/plugins/controls/public/control_group/init_controls_manager.test.ts b/src/plugins/controls/public/control_group/init_controls_manager.test.ts index 29998325664bb..d88dc5452a0e5 100644 --- a/src/plugins/controls/public/control_group/init_controls_manager.test.ts +++ b/src/plugins/controls/public/control_group/init_controls_manager.test.ts @@ -263,7 +263,7 @@ describe('getNewControlState', () => { test('should contain defaults when there are no existing controls', () => { const controlsManager = initControlsManager({}, new BehaviorSubject({})); expect(controlsManager.getNewControlState()).toEqual({ - grow: true, + grow: false, width: 'medium', dataViewId: undefined, }); @@ -284,7 +284,7 @@ describe('getNewControlState', () => { new BehaviorSubject(intialControlsState) ); expect(controlsManager.getNewControlState()).toEqual({ - grow: true, + grow: false, width: 'medium', dataViewId: 'myOtherDataViewId', }); diff --git a/src/plugins/controls/public/control_group/open_edit_control_group_flyout.tsx b/src/plugins/controls/public/control_group/open_edit_control_group_flyout.tsx index 54e35ab271b34..459913d98de0b 100644 --- a/src/plugins/controls/public/control_group/open_edit_control_group_flyout.tsx +++ b/src/plugins/controls/public/control_group/open_edit_control_group_flyout.tsx @@ -101,6 +101,9 @@ export const openEditControlGroupFlyout = ( 'aria-label': i18n.translate('controls.controlGroup.manageControl', { defaultMessage: 'Edit control settings', }), + size: 'm', + maxWidth: 500, + paddingSize: 'm', outsideClickCloses: false, onClose: () => closeOverlay(overlay), } diff --git a/src/plugins/controls/public/controls/data_controls/data_control_constants.tsx b/src/plugins/controls/public/controls/data_controls/data_control_constants.tsx index 6b06bd8a52439..23d4c68f6c5dc 100644 --- a/src/plugins/controls/public/controls/data_controls/data_control_constants.tsx +++ b/src/plugins/controls/public/controls/data_controls/data_control_constants.tsx @@ -21,14 +21,6 @@ export const DataControlEditorStrings = { defaultMessage: 'Edit control', }), dataSource: { - getFormGroupTitle: () => - i18n.translate('controls.controlGroup.manageControl.dataSource.formGroupTitle', { - defaultMessage: 'Data source', - }), - getFormGroupDescription: () => - i18n.translate('controls.controlGroup.manageControl.dataSource.formGroupDescription', { - defaultMessage: 'Select the data view and field that you want to create a control for.', - }), getSelectDataViewMessage: () => i18n.translate('controls.controlGroup.manageControl.dataSource.selectDataViewMessage', { defaultMessage: 'Please select a data view', @@ -95,14 +87,6 @@ export const DataControlEditorStrings = { }, }, displaySettings: { - getFormGroupTitle: () => - i18n.translate('controls.controlGroup.manageControl.displaySettings.formGroupTitle', { - defaultMessage: 'Display settings', - }), - getFormGroupDescription: () => - i18n.translate('controls.controlGroup.manageControl.displaySettings.formGroupDescription', { - defaultMessage: 'Change how the control appears on your dashboard.', - }), getTitleInputTitle: () => i18n.translate('controls.controlGroup.manageControl.displaySettings.titleInputTitle', { defaultMessage: 'Label', @@ -133,7 +117,7 @@ export const DataControlEditorStrings = { }, getSaveChangesTitle: () => i18n.translate('controls.controlGroup.manageControl.saveChangesTitle', { - defaultMessage: 'Save and close', + defaultMessage: 'Save', }), getCancelTitle: () => i18n.translate('controls.controlGroup.manageControl.cancelTitle', { diff --git a/src/plugins/controls/public/controls/data_controls/data_control_editor.tsx b/src/plugins/controls/public/controls/data_controls/data_control_editor.tsx index 23fd95978ff82..a84425f350dc1 100644 --- a/src/plugins/controls/public/controls/data_controls/data_control_editor.tsx +++ b/src/plugins/controls/public/controls/data_controls/data_control_editor.tsx @@ -15,7 +15,6 @@ import { EuiButtonEmpty, EuiButtonGroup, EuiCallOut, - EuiDescribedFormGroup, EuiFieldText, EuiFlexGroup, EuiFlexItem, @@ -250,20 +249,8 @@ export const DataControlEditor = - {DataControlEditorStrings.manageControl.controlTypeSettings.getFormGroupTitle( - controlFactory.getDisplayName() - )} - - } - description={DataControlEditorStrings.manageControl.controlTypeSettings.getFormGroupDescription( - controlFactory.getDisplayName() - )} - data-test-subj="control-editor-custom-settings" - > +
+ - +
); }, [fieldRegistry, controlFactory, initialState, editorState, controlGroupApi]); return ( <> - +

{!controlId // if no ID, then we are creating a new control ? DataControlEditorStrings.manageControl.getFlyoutCreateTitle() @@ -288,156 +275,144 @@ export const DataControlEditor = - {DataControlEditorStrings.manageControl.dataSource.getFormGroupTitle()}

} - description={DataControlEditorStrings.manageControl.dataSource.getFormGroupDescription()} - > - {!editorConfig?.hideDataViewSelector && ( - - {dataViewListError ? ( - -

{dataViewListError.message}

- - ) : ( - { - setEditorState({ ...editorState, dataViewId: newDataViewId }); - setSelectedControlType(undefined); - }} - trigger={{ - label: - selectedDataView?.getName() ?? - DataControlEditorStrings.manageControl.dataSource.getSelectDataViewMessage(), - }} - selectableProps={{ isLoading: dataViewListLoading }} - /> - )} - - )} - - - {fieldListError ? ( + {!editorConfig?.hideDataViewSelector && ( + + {dataViewListError ? ( -

{fieldListError.message}

+

{dataViewListError.message}

 ) : ( - { - const customPredicate = editorConfig?.fieldFilterPredicate?.(field) ?? true; - return Boolean(fieldRegistry?.[field.name]) && customPredicate; + { + setEditorState({ ...editorState, dataViewId: newDataViewId }); + setSelectedControlType(undefined); }} - selectedFieldName={editorState.fieldName} - dataView={selectedDataView} - onSelectField={(field) => { - setEditorState({ ...editorState, fieldName: field.name }); - - /** - * make sure that the new field is compatible with the selected control type and, if it's not, - * reset the selected control type to the **first** compatible control type - */ - const newCompatibleControlTypes = - fieldRegistry?.[field.name]?.compatibleControlTypes ?? []; - if ( - !selectedControlType || - !newCompatibleControlTypes.includes(selectedControlType!) - ) { - setSelectedControlType(newCompatibleControlTypes[0]); - } - - /** - * set the control title (i.e. the one set by the user) + default title (i.e. the field display name) - */ - const newDefaultTitle = field.displayName ?? field.name; - setDefaultPanelTitle(newDefaultTitle); - const currentTitle = editorState.title; - if (!currentTitle || currentTitle === newDefaultTitle) { - setPanelTitle(newDefaultTitle); - } - - setControlOptionsValid(true); // reset options state + trigger={{ + label: + selectedDataView?.getName() ?? + DataControlEditorStrings.manageControl.dataSource.getSelectDataViewMessage(), }} - selectableProps={{ isLoading: dataViewListLoading || dataViewLoading }} + selectableProps={{ isLoading: dataViewListLoading }} /> )} + )} + + + {fieldListError ? ( + +

{fieldListError.message}

+ + ) : ( + { + const customPredicate = editorConfig?.fieldFilterPredicate?.(field) ?? true; + return Boolean(fieldRegistry?.[field.name]) && customPredicate; + }} + selectedFieldName={editorState.fieldName} + dataView={selectedDataView} + onSelectField={(field) => { + setEditorState({ ...editorState, fieldName: field.name }); + + /** + * make sure that the new field is compatible with the selected control type and, if it's not, + * reset the selected control type to the **first** compatible control type + */ + const newCompatibleControlTypes = + fieldRegistry?.[field.name]?.compatibleControlTypes ?? []; + if ( + !selectedControlType || + !newCompatibleControlTypes.includes(selectedControlType!) + ) { + setSelectedControlType(newCompatibleControlTypes[0]); + } + + /** + * set the control title (i.e. the one set by the user) + default title (i.e. the field display name) + */ + const newDefaultTitle = field.displayName ?? field.name; + setDefaultPanelTitle(newDefaultTitle); + const currentTitle = editorState.title; + if (!currentTitle || currentTitle === newDefaultTitle) { + setPanelTitle(newDefaultTitle); + } + + setControlOptionsValid(true); // reset options state + }} + selectableProps={{ isLoading: dataViewListLoading || dataViewLoading }} + /> + )} + + + {/* wrapping in `div` so that focus gets passed properly to the form row */} +
+ +
+ + + { + setPanelTitle(e.target.value ?? ''); + setEditorState({ + ...editorState, + title: e.target.value === '' ? undefined : e.target.value, + }); + }} + /> + + {!editorConfig?.hideWidthSettings && ( - {/* wrapping in `div` so that focus gets passed properly to the form row */}
- + setEditorState({ ...editorState, width: newWidth as ControlWidth }) + } + /> + + setEditorState({ ...editorState, grow: !editorState.grow })} + data-test-subj="control-editor-grow-switch" />
- - {DataControlEditorStrings.manageControl.displaySettings.getFormGroupTitle()} - } - description={DataControlEditorStrings.manageControl.displaySettings.getFormGroupDescription()} - > - - { - setPanelTitle(e.target.value ?? ''); - setEditorState({ - ...editorState, - title: e.target.value === '' ? undefined : e.target.value, - }); - }} - /> - - {!editorConfig?.hideWidthSettings && ( - -
- - setEditorState({ ...editorState, width: newWidth as ControlWidth }) - } - /> - - setEditorState({ ...editorState, grow: !editorState.grow })} - data-test-subj="control-editor-grow-switch" - /> -
- - )} - + )} {!editorConfig?.hideAdditionalSettings && CustomSettingsComponent} {controlId && ( <> @@ -464,7 +439,6 @@ export const DataControlEditor = { onCancel(editorState); }} @@ -476,7 +450,7 @@ export const DataControlEditor = closeOverlay(overlay), } ); diff --git a/src/plugins/controls/public/controls/data_controls/options_list_control/components/options_list_editor_options.tsx b/src/plugins/controls/public/controls/data_controls/options_list_control/components/options_list_editor_options.tsx index e9dad12be5623..f07a7cc6c58bf 100644 --- a/src/plugins/controls/public/controls/data_controls/options_list_control/components/options_list_editor_options.tsx +++ b/src/plugins/controls/public/controls/data_controls/options_list_control/components/options_list_editor_options.tsx @@ -131,6 +131,7 @@ export const OptionsListEditorOptions = ({ data-test-subj="optionsListControl__selectionOptionsRadioGroup" > { @@ -146,6 +147,7 @@ export const OptionsListEditorOptions = ({ data-test-subj="optionsListControl__searchOptionsRadioGroup" > { @@ -158,6 +160,7 @@ export const OptionsListEditorOptions = ({ )} { const newStep = event.target.valueAsNumber; diff --git a/src/plugins/dashboard/public/dashboard_app/top_nav/add_new_panel/dashboard_panel_selection_flyout.tsx b/src/plugins/dashboard/public/dashboard_app/top_nav/add_new_panel/dashboard_panel_selection_flyout.tsx index dbb86046def06..e6adece8ab36d 100644 --- a/src/plugins/dashboard/public/dashboard_app/top_nav/add_new_panel/dashboard_panel_selection_flyout.tsx +++ b/src/plugins/dashboard/public/dashboard_app/top_nav/add_new_panel/dashboard_panel_selection_flyout.tsx @@ -125,7 +125,7 @@ export const DashboardPanelSelectionListFlyout: React.FC< return ( <> - +

{ @@ -281,7 +282,7 @@ export const DashboardPanelSelectionListFlyout: React.FC< diff --git a/src/plugins/dashboard/public/dashboard_app/top_nav/editor_menu.tsx b/src/plugins/dashboard/public/dashboard_app/top_nav/editor_menu.tsx index 2cad63c442026..cf7f9c65c6618 100644 --- a/src/plugins/dashboard/public/dashboard_app/top_nav/editor_menu.tsx +++ b/src/plugins/dashboard/public/dashboard_app/top_nav/editor_menu.tsx @@ -43,7 +43,7 @@ export const EditorMenu = ({ createNewVisType, isDisabled }: EditorMenuProps) => function openDashboardPanelSelectionFlyout() { const flyoutPanelPaddingSize: ComponentProps< typeof DashboardPanelSelectionListFlyout - >['paddingSize'] = 'l'; + >['paddingSize'] = 'm'; const mount = toMountPoint( React.createElement(function () { diff --git a/src/plugins/embeddable/public/add_panel_flyout/add_panel_flyout.tsx b/src/plugins/embeddable/public/add_panel_flyout/add_panel_flyout.tsx index b83ebbcb49d66..b334dbcb5857a 100644 --- a/src/plugins/embeddable/public/add_panel_flyout/add_panel_flyout.tsx +++ b/src/plugins/embeddable/public/add_panel_flyout/add_panel_flyout.tsx @@ -189,7 +189,7 @@ export const AddPanelFlyout = ({ return ( <> - +

{i18n.translate('embeddableApi.addPanel.Title', { defaultMessage: 'Add from library' })}

diff --git a/src/plugins/embeddable/public/add_panel_flyout/open_add_panel_flyout.tsx b/src/plugins/embeddable/public/add_panel_flyout/open_add_panel_flyout.tsx index 160289d0d1c2a..9ba3c00a73745 100644 --- a/src/plugins/embeddable/public/add_panel_flyout/open_add_panel_flyout.tsx +++ b/src/plugins/embeddable/public/add_panel_flyout/open_add_panel_flyout.tsx @@ -52,6 +52,9 @@ export const openAddPanelFlyout = ({ if (onClose) onClose(); overlayRef.close(); }, + size: 'm', + maxWidth: 500, + paddingSize: 'm', 'data-test-subj': 'dashboardAddPanel', 'aria-labelledby': modalTitleId, } diff --git a/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.test.tsx b/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.test.tsx index 265f162d04f6c..f052f0526a945 100644 --- a/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.test.tsx +++ b/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.test.tsx @@ -43,11 +43,11 @@ const ImageEditor = (props: Partial) => { ); }; -test('should call onCancel when "Close" clicked', async () => { +test('should call onCancel when "Cancel" clicked', async () => { const onCancel = jest.fn(); const { getByText } = render(); - expect(getByText('Close')).toBeVisible(); - await userEvent.click(getByText('Close')); + expect(getByText('Cancel')).toBeVisible(); + await userEvent.click(getByText('Cancel')); expect(onCancel).toBeCalled(); }); diff --git a/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.tsx b/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.tsx index 2c57f25db6c8b..1a5ee3bc64e1d 100644 --- a/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.tsx +++ b/src/plugins/image_embeddable/public/components/image_editor/image_editor_flyout.tsx @@ -121,7 +121,7 @@ export function ImageEditorFlyout(props: ImageEditorFlyoutProps) { return ( <> - +

{isEditing ? ( - - + + + setSrcType('file')} isSelected={srcType === 'file'}> - - + {srcType === 'file' && ( <> {isDraftImageConfigValid ? ( @@ -238,7 +238,7 @@ export function ImageEditorFlyout(props: ImageEditorFlyoutProps) { />

} - titleSize={'s'} + titleSize={'xs'} /> ) : ( )} - - + )} - - - - - - + diff --git a/src/plugins/image_embeddable/public/components/image_editor/open_image_editor.tsx b/src/plugins/image_embeddable/public/components/image_editor/open_image_editor.tsx index ae8ced88d14ef..f730147cb0d2c 100644 --- a/src/plugins/image_embeddable/public/components/image_editor/open_image_editor.tsx +++ b/src/plugins/image_embeddable/public/components/image_editor/open_image_editor.tsx @@ -79,6 +79,9 @@ export const openImageEditor = async ({ onClose: () => { onCancel(); }, + size: 'm', + maxWidth: 500, + paddingSize: 'm', ownFocus: true, 'data-test-subj': 'createImageEmbeddableFlyout', } diff --git a/src/plugins/links/public/components/dashboard_link/dashboard_link_destination_picker.tsx b/src/plugins/links/public/components/dashboard_link/dashboard_link_destination_picker.tsx index b062b9befa284..ab5b923327049 100644 --- a/src/plugins/links/public/components/dashboard_link/dashboard_link_destination_picker.tsx +++ b/src/plugins/links/public/components/dashboard_link/dashboard_link_destination_picker.tsx @@ -119,6 +119,7 @@ export const DashboardLinkDestinationPicker = ({ return ( onClose()} > - +

{link ? LinksStrings.editor.getEditLinkTitle() @@ -113,6 +113,7 @@ export const LinkEditor = ({ { @@ -131,6 +132,7 @@ export const LinkEditor = ({ /> onClose()} - iconType="cross" data-test-subj="links--linkEditor--closeBtn" > {LinksStrings.editor.getCancelButtonLabel()} @@ -160,6 +162,7 @@ export const LinkEditor = ({ { // this check should always be true, since the button is disabled otherwise - this is just for type safety diff --git a/src/plugins/links/public/components/editor/links_editor.scss b/src/plugins/links/public/components/editor/links_editor.scss index 02961c7d5f5cb..c33b95350df98 100644 --- a/src/plugins/links/public/components/editor/links_editor.scss +++ b/src/plugins/links/public/components/editor/links_editor.scss @@ -3,7 +3,7 @@ .linksPanelEditor { .linkEditor { @include euiFlyout; - max-inline-size: $euiSizeXXL * 18; // 40px * 18 = 720px + max-inline-size: $euiSizeXS * 125; // 4px * 125 = 500px &.in { animation: euiFlyoutOpenAnimation $euiAnimSpeedNormal $euiAnimSlightResistance; @@ -59,6 +59,9 @@ } .links_hoverActions { + background-color: $euiColorEmptyShade; + position: absolute; + right: $euiSizeL; opacity: 0; visibility: hidden; transition: visibility $euiAnimSpeedNormal, opacity $euiAnimSpeedNormal; diff --git a/src/plugins/links/public/components/editor/links_editor.tsx b/src/plugins/links/public/components/editor/links_editor.tsx index 93ca47e364c57..8fa33fd4ebcaa 100644 --- a/src/plugins/links/public/components/editor/links_editor.tsx +++ b/src/plugins/links/public/components/editor/links_editor.tsx @@ -167,7 +167,7 @@ const LinksEditor = ({ - +

{isEditingExisting ? LinksStrings.editor.panelEditor.getEditFlyoutTitle() @@ -251,7 +251,6 @@ const LinksEditor = ({ @@ -268,6 +267,7 @@ const LinksEditor = ({ data-test-subj="links--panelEditor--saveByReferenceTooltip" > - - - - - - - - - - - - - - + + + + + + + + + + + + ); diff --git a/src/plugins/links/public/components/external_link/external_link_destination_picker.tsx b/src/plugins/links/public/components/external_link/external_link_destination_picker.tsx index dfdc6e0589e6b..5b8522b39960e 100644 --- a/src/plugins/links/public/components/external_link/external_link_destination_picker.tsx +++ b/src/plugins/links/public/components/external_link/external_link_destination_picker.tsx @@ -54,6 +54,7 @@ export const ExternalLinkDestinationPicker = ({ return (
i18n.translate('links.editor.cancelButtonLabel', { - defaultMessage: 'Close', + defaultMessage: 'Cancel', }), panelEditor: { getLinksTitle: () => diff --git a/src/plugins/links/public/editor/open_editor_flyout.tsx b/src/plugins/links/public/editor/open_editor_flyout.tsx index 041672e89dbbe..87b1ab4e21ff8 100644 --- a/src/plugins/links/public/editor/open_editor_flyout.tsx +++ b/src/plugins/links/public/editor/open_editor_flyout.tsx @@ -137,7 +137,8 @@ export async function openEditorFlyout({ ), { id: flyoutId, - maxWidth: 720, + maxWidth: 500, + paddingSize: 'm', ownFocus: true, onClose: onCancel, outsideClickCloses: false, diff --git a/src/plugins/presentation_util/public/components/dashboard_drilldown_options/dashboard_drilldown_options.tsx b/src/plugins/presentation_util/public/components/dashboard_drilldown_options/dashboard_drilldown_options.tsx index 921560f7a2224..63ff89da2ec17 100644 --- a/src/plugins/presentation_util/public/components/dashboard_drilldown_options/dashboard_drilldown_options.tsx +++ b/src/plugins/presentation_util/public/components/dashboard_drilldown_options/dashboard_drilldown_options.tsx @@ -8,7 +8,7 @@ */ import React from 'react'; -import { EuiFormRow, EuiSwitch } from '@elastic/eui'; +import { EuiFormRow, EuiSpacer, EuiSwitch } from '@elastic/eui'; import { DashboardDrilldownOptions } from './types'; import { dashboardDrilldownConfigStrings } from '../../i18n/dashboard_drilldown_config'; @@ -24,32 +24,35 @@ export const DashboardDrilldownOptionsComponent = ({ }: DashboardDrilldownOptionsProps) => { return ( <> - - onOptionChange({ useCurrentFilters: !options.useCurrentFilters })} - data-test-subj="dashboardDrillDownOptions--useCurrentFilters--checkbox" - /> - - - onOptionChange({ useCurrentDateRange: !options.useCurrentDateRange })} - data-test-subj="dashboardDrillDownOptions--useCurrentDateRange--checkbox" - /> - - - onOptionChange({ openInNewTab: !options.openInNewTab })} - data-test-subj="dashboardDrillDownOptions--openInNewTab--checkbox" - /> + +
+ onOptionChange({ useCurrentFilters: !options.useCurrentFilters })} + data-test-subj="dashboardDrillDownOptions--useCurrentFilters--checkbox" + /> + + onOptionChange({ useCurrentDateRange: !options.useCurrentDateRange })} + data-test-subj="dashboardDrillDownOptions--useCurrentDateRange--checkbox" + /> + + onOptionChange({ openInNewTab: !options.openInNewTab })} + data-test-subj="dashboardDrillDownOptions--openInNewTab--checkbox" + /> +
); diff --git a/src/plugins/presentation_util/public/components/data_view_picker/data_view_picker.tsx b/src/plugins/presentation_util/public/components/data_view_picker/data_view_picker.tsx index e985e9bec357a..1c8466097bd98 100644 --- a/src/plugins/presentation_util/public/components/data_view_picker/data_view_picker.tsx +++ b/src/plugins/presentation_util/public/components/data_view_picker/data_view_picker.tsx @@ -52,6 +52,7 @@ export function DataViewPicker({ data-test-subj="open-data-view-picker" onClick={() => setPopoverIsOpen(!isPopoverOpen)} label={label} + size="s" fullWidth {...colorProp} {...rest} diff --git a/src/plugins/presentation_util/public/components/field_picker/field_picker.tsx b/src/plugins/presentation_util/public/components/field_picker/field_picker.tsx index daac202f21b66..0b81cfd66156d 100644 --- a/src/plugins/presentation_util/public/components/field_picker/field_picker.tsx +++ b/src/plugins/presentation_util/public/components/field_picker/field_picker.tsx @@ -140,6 +140,7 @@ export const FieldPicker = ({ placeholder: i18n.translate('presentationUtil.fieldSearch.searchPlaceHolder', { defaultMessage: 'Search field names', }), + compressed: true, disabled: Boolean(selectableProps?.isLoading), inputRef: setSearchRef, }} diff --git a/src/plugins/presentation_util/public/components/field_picker/field_type_filter.tsx b/src/plugins/presentation_util/public/components/field_picker/field_type_filter.tsx index d2e929b8a9a84..4212668599a0d 100644 --- a/src/plugins/presentation_util/public/components/field_picker/field_type_filter.tsx +++ b/src/plugins/presentation_util/public/components/field_picker/field_type_filter.tsx @@ -63,7 +63,7 @@ export function FieldTypeFilter({ ); return ( - + { return ( <> - - onOptionChange({ openInNewTab: !options.openInNewTab })} - data-test-subj="urlDrilldownOpenInNewTab" - /> - - - - {txtUrlTemplateEncodeUrl} - - {txtUrlTemplateEncodeDescription} - - } - checked={options.encodeUrl} - onChange={() => onOptionChange({ encodeUrl: !options.encodeUrl })} - data-test-subj="urlDrilldownEncodeUrl" - /> + +
+ onOptionChange({ openInNewTab: !options.openInNewTab })} + data-test-subj="urlDrilldownOpenInNewTab" + /> + + + {txtUrlTemplateEncodeUrl} + + {txtUrlTemplateEncodeDescription} + + } + checked={options.encodeUrl} + onChange={() => onOptionChange({ encodeUrl: !options.encodeUrl })} + data-test-subj="urlDrilldownEncodeUrl" + /> +
); diff --git a/test/api_integration/apis/data_views/fields_for_wildcard_route/response.ts b/test/api_integration/apis/data_views/fields_for_wildcard_route/response.ts index 2b0fa0dab4424..a80ca89ee4865 100644 --- a/test/api_integration/apis/data_views/fields_for_wildcard_route/response.ts +++ b/test/api_integration/apis/data_views/fields_for_wildcard_route/response.ts @@ -80,7 +80,8 @@ export default function ({ getService }: FtrProviderContext) { }, ]; - describe('fields_for_wildcard_route response', () => { + // Failing: See https://github.com/elastic/kibana/issues/199413 + describe.skip('fields_for_wildcard_route response', () => { before(() => esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') ); diff --git a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/install_elser.ts b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/install_elser.ts index 037a9e809d1e1..09dc85b816191 100644 --- a/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/install_elser.ts +++ b/x-pack/packages/ai-infra/product-doc-artifact-builder/src/tasks/install_elser.ts @@ -60,7 +60,6 @@ const waitUntilDeployed = async ({ model_id: modelId, }); const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats; - // @ts-expect-error deploymentStats.nodes not defined as array even if it is. if (!deploymentStats || deploymentStats.nodes.length === 0) { await sleep(delay); continue; diff --git a/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts b/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts index 3d37331b4cc5d..076c685aca5b9 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts +++ b/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts @@ -6,14 +6,26 @@ */ import { schema } from '@kbn/config-schema'; +import { ApiMessageCode } from '../../types/graph/v1'; export const graphRequestSchema = schema.object({ + nodesLimit: schema.maybe(schema.number()), + showUnknownTarget: schema.maybe(schema.boolean()), query: schema.object({ - actorIds: schema.arrayOf(schema.string()), eventIds: schema.arrayOf(schema.string()), // TODO: use zod for range validation instead of config schema start: schema.oneOf([schema.number(), schema.string()]), end: schema.oneOf([schema.number(), schema.string()]), + esQuery: schema.maybe( + schema.object({ + bool: schema.object({ + filter: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + must: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + should: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + must_not: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + }), + }) + ), }), }); @@ -23,6 +35,9 @@ export const graphResponseSchema = () => schema.oneOf([entityNodeDataSchema, groupNodeDataSchema, labelNodeDataSchema]) ), edges: schema.arrayOf(edgeDataSchema), + messages: schema.maybe( + schema.arrayOf(schema.oneOf([schema.literal(ApiMessageCode.ReachedNodesLimit)])) + ), }); export const colorSchema = schema.oneOf([ diff --git a/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json b/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json index c7cf1e9208bfc..ebec9929559f0 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json +++ b/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json @@ -20,5 +20,6 @@ "@kbn/i18n", "@kbn/analytics", "@kbn/usage-collection-plugin", + "@kbn/es-query", ] } diff --git a/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts b/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts index 48d1d1c49fd03..f97d11b34732c 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts +++ b/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts @@ -6,6 +6,7 @@ */ import type { TypeOf } from '@kbn/config-schema'; +import type { BoolQuery } from '@kbn/es-query'; import { colorSchema, edgeDataSchema, @@ -17,13 +18,21 @@ import { nodeShapeSchema, } from '../../schema/graph/v1'; -export type GraphRequest = TypeOf; -export type GraphResponse = TypeOf; +export type GraphRequest = Omit, 'query.esQuery'> & { + query: { esQuery?: { bool: Partial } }; +}; +export type GraphResponse = Omit, 'messages'> & { + messages?: ApiMessageCode[]; +}; export type Color = typeof colorSchema.type; export type NodeShape = TypeOf; +export enum ApiMessageCode { + ReachedNodesLimit = 'REACHED_NODES_LIMIT', +} + export type EntityNodeDataModel = TypeOf; export type GroupNodeDataModel = TypeOf; diff --git a/x-pack/packages/ml/random_sampler_utils/src/random_sampler_wrapper.ts b/x-pack/packages/ml/random_sampler_utils/src/random_sampler_wrapper.ts index 5054833ac7dd0..39d26509422a2 100644 --- a/x-pack/packages/ml/random_sampler_utils/src/random_sampler_wrapper.ts +++ b/x-pack/packages/ml/random_sampler_utils/src/random_sampler_wrapper.ts @@ -69,7 +69,6 @@ export const createRandomSamplerWrapper = (options: RandomSamplerOptions) => { return { [aggName]: { - // @ts-expect-error `random_sampler` is not yet part of `AggregationsAggregationContainer` random_sampler: { probability, ...(options.seed ? { seed: options.seed } : {}), diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts index 9e9744b33d940..9fb817b275a0d 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts @@ -10,6 +10,7 @@ import { graphResponseSchema, } from '@kbn/cloud-security-posture-common/schema/graph/latest'; import { transformError } from '@kbn/securitysolution-es-utils'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/v1'; import { GRAPH_ROUTE_PATH } from '../../../common/constants'; import { CspRouter } from '../../types'; import { getGraph as getGraphV1 } from './v1'; @@ -39,26 +40,29 @@ export const defineGraphRoute = (router: CspRouter) => }, }, async (context, request, response) => { - const { actorIds, eventIds, start, end } = request.body.query; + const { nodesLimit, showUnknownTarget = false } = request.body; + const { eventIds, start, end, esQuery } = request.body.query as GraphRequest['query']; const cspContext = await context.csp; const spaceId = (await cspContext.spaces?.spacesService?.getActiveSpace(request))?.id; try { - const { nodes, edges } = await getGraphV1( - { + const resp = await getGraphV1({ + services: { logger: cspContext.logger, esClient: cspContext.esClient, }, - { - actorIds, + query: { eventIds, spaceId, start, end, - } - ); + esQuery, + }, + showUnknownTarget, + nodesLimit, + }); - return response.ok({ body: { nodes, edges } }); + return response.ok({ body: resp }); } catch (err) { const error = transformError(err); cspContext.logger.error(`Failed to fetch graph ${err}`); diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts deleted file mode 100644 index ba32664da6233..0000000000000 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - EdgeDataModel, - NodeDataModel, -} from '@kbn/cloud-security-posture-common/types/graph/latest'; -import type { Logger, IScopedClusterClient } from '@kbn/core/server'; -import type { Writable } from '@kbn/utility-types'; - -export interface GraphContextServices { - logger: Logger; - esClient: IScopedClusterClient; -} - -export interface GraphContext { - nodes: Array>; - edges: Array>; -} diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts index 5102d153c1905..b14a2ba3e06a9 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts @@ -8,22 +8,27 @@ import { castArray } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import type { Logger, IScopedClusterClient } from '@kbn/core/server'; +import { ApiMessageCode } from '@kbn/cloud-security-posture-common/types/graph/latest'; import type { + Color, EdgeDataModel, - NodeDataModel, EntityNodeDataModel, - LabelNodeDataModel, + GraphRequest, + GraphResponse, GroupNodeDataModel, -} from '@kbn/cloud-security-posture-common/types/graph/latest'; + LabelNodeDataModel, + NodeDataModel, +} from '@kbn/cloud-security-posture-common/types/graph/v1'; import type { EsqlToRecords } from '@elastic/elasticsearch/lib/helpers'; import type { Writable } from '@kbn/utility-types'; -import type { GraphContextServices, GraphContext } from './types'; + +type EsQuery = GraphRequest['query']['esQuery']; interface GraphEdge { badge: number; - ips: string[]; - hosts: string[]; - users: string[]; + ips?: string[] | string; + hosts?: string[] | string; + users?: string[] | string; actorIds: string[] | string; action: string; targetIds: string[] | string; @@ -36,50 +41,75 @@ interface LabelEdges { target: string; } -export const getGraph = async ( - services: GraphContextServices, +interface GraphContextServices { + logger: Logger; + esClient: IScopedClusterClient; +} + +interface GetGraphParams { + services: GraphContextServices; query: { - actorIds: string[]; eventIds: string[]; spaceId?: string; start: string | number; end: string | number; - } -): Promise<{ - nodes: NodeDataModel[]; - edges: EdgeDataModel[]; -}> => { - const { esClient, logger } = services; - const { actorIds, eventIds, spaceId = 'default', start, end } = query; - - logger.trace( - `Fetching graph for [eventIds: ${eventIds.join(', ')}] [actorIds: ${actorIds.join( - ', ' - )}] in [spaceId: ${spaceId}]` - ); + esQuery?: EsQuery; + }; + showUnknownTarget: boolean; + nodesLimit?: number; +} - const results = await fetchGraph({ esClient, logger, start, end, eventIds, actorIds }); +export const getGraph = async ({ + services: { esClient, logger }, + query: { eventIds, spaceId = 'default', start, end, esQuery }, + showUnknownTarget, + nodesLimit, +}: GetGraphParams): Promise> => { + logger.trace(`Fetching graph for [eventIds: ${eventIds.join(', ')}] in [spaceId: ${spaceId}]`); + + const results = await fetchGraph({ + esClient, + showUnknownTarget, + logger, + start, + end, + eventIds, + esQuery, + }); // Convert results into set of nodes and edges - const graphContext = parseRecords(logger, results.records); - - return { nodes: graphContext.nodes, edges: graphContext.edges }; + return parseRecords(logger, results.records, nodesLimit); }; interface ParseContext { - nodesMap: Record; - edgesMap: Record; - edgeLabelsNodes: Record; - labelEdges: Record; + readonly nodesLimit?: number; + readonly nodesMap: Record; + readonly edgesMap: Record; + readonly edgeLabelsNodes: Record; + readonly labelEdges: Record; + readonly messages: ApiMessageCode[]; + readonly logger: Logger; } -const parseRecords = (logger: Logger, records: GraphEdge[]): GraphContext => { - const ctx: ParseContext = { nodesMap: {}, edgeLabelsNodes: {}, edgesMap: {}, labelEdges: {} }; +const parseRecords = ( + logger: Logger, + records: GraphEdge[], + nodesLimit?: number +): Pick => { + const ctx: ParseContext = { + nodesLimit, + logger, + nodesMap: {}, + edgeLabelsNodes: {}, + edgesMap: {}, + labelEdges: {}, + messages: [], + }; - logger.trace(`Parsing records [length: ${records.length}]`); + logger.trace(`Parsing records [length: ${records.length}] [nodesLimit: ${nodesLimit ?? 'none'}]`); - createNodes(logger, records, ctx); - createEdgesAndGroups(logger, ctx); + createNodes(records, ctx); + createEdgesAndGroups(ctx); logger.trace( `Parsed [nodes: ${Object.keys(ctx.nodesMap).length}, edges: ${ @@ -90,7 +120,11 @@ const parseRecords = (logger: Logger, records: GraphEdge[]): GraphContext => { // Sort groups to be first (fixes minor layout issue) const nodes = sortNodes(ctx.nodesMap); - return { nodes, edges: Object.values(ctx.edgesMap) }; + return { + nodes, + edges: Object.values(ctx.edgesMap), + messages: ctx.messages.length > 0 ? ctx.messages : undefined, + }; }; const fetchGraph = async ({ @@ -98,15 +132,17 @@ const fetchGraph = async ({ logger, start, end, - actorIds, eventIds, + showUnknownTarget, + esQuery, }: { esClient: IScopedClusterClient; logger: Logger; start: string | number; end: string | number; - actorIds: string[]; eventIds: string[]; + showUnknownTarget: boolean; + esQuery?: EsQuery; }): Promise> => { const query = `from logs-* | WHERE event.action IS NOT NULL AND actor.entity.id IS NOT NULL @@ -124,59 +160,84 @@ const fetchGraph = async ({ targetIds = target.entity.id, eventOutcome = event.outcome, isAlert -| LIMIT 1000`; +| LIMIT 1000 +| SORT isAlert DESC`; logger.trace(`Executing query [${query}]`); return await esClient.asCurrentUser.helpers .esql({ columnar: false, - filter: { - bool: { - must: [ + filter: buildDslFilter(eventIds, showUnknownTarget, start, end, esQuery), + query, + // @ts-ignore - types are not up to date + params: [...eventIds.map((id, idx) => ({ [`al_id${idx}`]: id }))], + }) + .toRecords(); +}; + +const buildDslFilter = ( + eventIds: string[], + showUnknownTarget: boolean, + start: string | number, + end: string | number, + esQuery?: EsQuery +) => ({ + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: start, + lte: end, + }, + }, + }, + ...(showUnknownTarget + ? [] + : [ { - range: { - '@timestamp': { - gte: start, - lte: end, - }, + exists: { + field: 'target.entity.id', }, }, + ]), + { + bool: { + should: [ + ...(esQuery?.bool.filter?.length || + esQuery?.bool.must?.length || + esQuery?.bool.should?.length || + esQuery?.bool.must_not?.length + ? [esQuery] + : []), { - bool: { - should: [ - { - terms: { - 'event.id': eventIds, - }, - }, - { - terms: { - 'actor.entity.id': actorIds, - }, - }, - ], - minimum_should_match: 1, + terms: { + 'event.id': eventIds, }, }, ], + minimum_should_match: 1, }, }, - query, - // @ts-ignore - types are not up to date - params: [...eventIds.map((id, idx) => ({ [`al_id${idx}`]: id }))], - }) - .toRecords(); -}; + ], + }, +}); -const createNodes = ( - logger: Logger, - records: GraphEdge[], - context: Omit -) => { +const createNodes = (records: GraphEdge[], context: Omit) => { const { nodesMap, edgeLabelsNodes, labelEdges } = context; for (const record of records) { + if (context.nodesLimit !== undefined && Object.keys(nodesMap).length >= context.nodesLimit) { + context.logger.debug( + `Reached nodes limit [limit: ${context.nodesLimit}] [current: ${ + Object.keys(nodesMap).length + }]` + ); + context.messages.push(ApiMessageCode.ReachedNodesLimit); + break; + } + const { ips, hosts, users, actorIds, action, targetIds, isAlert, eventOutcome } = record; const actorIdsArray = castArray(actorIds); const targetIdsArray = castArray(targetIds); @@ -190,12 +251,6 @@ const createNodes = ( } }); - logger.trace( - `Parsing record [actorIds: ${actorIdsArray.join( - ', ' - )}, action: ${action}, targetIds: ${targetIdsArray.join(', ')}]` - ); - // Create entity nodes [...actorIdsArray, ...targetIdsArray].forEach((id) => { if (nodesMap[id] === undefined) { @@ -203,10 +258,13 @@ const createNodes = ( id, label: unknownTargets.includes(id) ? 'Unknown' : undefined, color: isAlert ? 'danger' : 'primary', - ...determineEntityNodeShape(id, ips ?? [], hosts ?? [], users ?? []), + ...determineEntityNodeShape( + id, + castArray(ips ?? []), + castArray(hosts ?? []), + castArray(users ?? []) + ), }; - - logger.trace(`Creating entity node [${id}]`); } }); @@ -226,8 +284,6 @@ const createNodes = ( shape: 'label', }; - logger.trace(`Creating label node [${labelNode.id}]`); - nodesMap[labelNode.id] = labelNode; edgeLabelsNodes[edgeId].push(labelNode.id); labelEdges[labelNode.id] = { source: actorId, target: targetId }; @@ -278,7 +334,7 @@ const sortNodes = (nodesMap: Record) => { return [...groupNodes, ...otherNodes]; }; -const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { +const createEdgesAndGroups = (context: ParseContext) => { const { edgeLabelsNodes, edgesMap, nodesMap, labelEdges } = context; Object.entries(edgeLabelsNodes).forEach(([edgeId, edgeLabelsIds]) => { @@ -287,7 +343,6 @@ const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { const edgeLabelId = edgeLabelsIds[0]; connectEntitiesAndLabelNode( - logger, edgesMap, nodesMap, labelEdges[edgeLabelId].source, @@ -300,44 +355,47 @@ const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { shape: 'group', }; nodesMap[groupNode.id] = groupNode; + let groupEdgesColor: Color = 'primary'; + + edgeLabelsIds.forEach((edgeLabelId) => { + (nodesMap[edgeLabelId] as Writable).parentId = groupNode.id; + connectEntitiesAndLabelNode(edgesMap, nodesMap, groupNode.id, edgeLabelId, groupNode.id); + + if ((nodesMap[edgeLabelId] as LabelNodeDataModel).color === 'danger') { + groupEdgesColor = 'danger'; + } else if ( + (nodesMap[edgeLabelId] as LabelNodeDataModel).color === 'warning' && + groupEdgesColor !== 'danger' + ) { + // Use warning only if there's no danger color + groupEdgesColor = 'warning'; + } + }); connectEntitiesAndLabelNode( - logger, edgesMap, nodesMap, labelEdges[edgeLabelsIds[0]].source, groupNode.id, - labelEdges[edgeLabelsIds[0]].target + labelEdges[edgeLabelsIds[0]].target, + groupEdgesColor ); - - edgeLabelsIds.forEach((edgeLabelId) => { - (nodesMap[edgeLabelId] as Writable).parentId = groupNode.id; - connectEntitiesAndLabelNode( - logger, - edgesMap, - nodesMap, - groupNode.id, - edgeLabelId, - groupNode.id - ); - }); } }); }; const connectEntitiesAndLabelNode = ( - logger: Logger, edgesMap: Record, nodesMap: Record, sourceNodeId: string, labelNodeId: string, - targetNodeId: string + targetNodeId: string, + colorOverride?: Color ) => { [ - connectNodes(nodesMap, sourceNodeId, labelNodeId), - connectNodes(nodesMap, labelNodeId, targetNodeId), + connectNodes(nodesMap, sourceNodeId, labelNodeId, colorOverride), + connectNodes(nodesMap, labelNodeId, targetNodeId, colorOverride), ].forEach((edge) => { - logger.trace(`Connecting nodes [${edge.source} -> ${edge.target}]`); edgesMap[edge.id] = edge; }); }; @@ -345,7 +403,8 @@ const connectEntitiesAndLabelNode = ( const connectNodes = ( nodesMap: Record, sourceNodeId: string, - targetNodeId: string + targetNodeId: string, + colorOverride?: Color ): EdgeDataModel => { const sourceNode = nodesMap[sourceNodeId]; const targetNode = nodesMap[targetNodeId]; @@ -360,6 +419,6 @@ const connectNodes = ( id: `a(${sourceNodeId})-b(${targetNodeId})`, source: sourceNodeId, target: targetNodeId, - color, + color: colorOverride ?? color, }; }; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts index 0a3632efe9195..abdffd19eca76 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_named_search_to_list_item.ts @@ -34,7 +34,7 @@ export const transformElasticNamedSearchToListItem = ({ }: TransformElasticMSearchToListItemOptions): SearchListItemArraySchema => { return value.map((singleValue, index) => { const matchingHits = response.hits.hits.filter((hit) => { - if (hit.matched_queries != null) { + if (hit.matched_queries != null && Array.isArray(hit.matched_queries)) { return hit.matched_queries.some((matchedQuery) => { const [matchedQueryIndex] = matchedQuery.split('.'); return matchedQueryIndex === `${index}`; diff --git a/x-pack/plugins/ml/public/application/model_management/expanded_row.tsx b/x-pack/plugins/ml/public/application/model_management/expanded_row.tsx index e9db6b4e4f590..f7e95e3eda52c 100644 --- a/x-pack/plugins/ml/public/application/model_management/expanded_row.tsx +++ b/x-pack/plugins/ml/public/application/model_management/expanded_row.tsx @@ -182,6 +182,7 @@ export const ExpandedRow: FC = ({ item }) => { key: `${perDeploymentStat.deployment_id}_${nodeName}`, ...perDeploymentStat, ...modelSizeStats, + // @ts-expect-error `throughput_last_minute` is not declared in ES Types node: { ...pick(n, [ 'average_inference_time_ms', diff --git a/x-pack/plugins/ml/server/models/model_management/memory_usage.ts b/x-pack/plugins/ml/server/models/model_management/memory_usage.ts index 6e2931dbbe06b..6e9121a133bb8 100644 --- a/x-pack/plugins/ml/server/models/model_management/memory_usage.ts +++ b/x-pack/plugins/ml/server/models/model_management/memory_usage.ts @@ -181,6 +181,7 @@ export class MemoryUsageService { const mlNodes = Object.entries(response.nodes).filter(([, node]) => node.roles.includes('ml')); + // @ts-expect-error `throughput_last_minute` is not declared in ES Types const nodeDeploymentStatsResponses: NodeDeploymentStatsResponse[] = mlNodes.map( ([nodeId, node]) => { const nodeFields = pick(node, NODE_FIELDS) as RequiredNodeFields; diff --git a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/exporting.md b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/exporting.md new file mode 100644 index 0000000000000..f4cbc66779a81 --- /dev/null +++ b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/exporting.md @@ -0,0 +1,65 @@ +# Prebuilt Rule Export + +This is a test plan for the exporting of prebuilt rules. This feature is an aspect of `Milestone 2` of the [Rule Immutability/Customization](https://github.com/elastic/security-team/issues/1974) epic. + +Status: `in progress`. + +## Useful information + +### Tickets + +- [Rule Immutability/Customization](https://github.com/elastic/security-team/issues/1974) +- [Rule Exporting Feature](https://github.com/elastic/kibana/issues/180167#issue-2227974379) +- [Rule Export API PR](https://github.com/elastic/kibana/pull/194498) + +### Terminology + +- **prebuilt rule**: A rule contained in our `Prebuilt Security Detection Rules` integration in Fleet. +- **custom rule**: A rule defined by the user, which has no relation to the prebuilt rules +- **rule source, or ruleSource**: A field on the rule that defines the rule's categorization + +## Scenarios + +### Core Functionality + +#### Scenario: Exporting prebuilt rule individually +```Gherkin +Given a space with prebuilt rules installed +When the user selects "Export rule" from the "All actions" dropdown on the rule's page +Then the rule should be exported as an NDJSON file +And it should include an "immutable" field with a value of true +And its "ruleSource" "type" should be "external" +And its "ruleSource" "isCustomized" value should depend on whether the rule was customized +``` + +#### Scenario: Exporting prebuilt rules in bulk +```Gherkin +Given a space with prebuilt rules installed +When the user selects prebuilt rules in the alerts table +And chooses "Export" from bulk actions +Then the selected rules should be exported as an NDJSON file +And they should include an "immutable" field with a value of true +And their "ruleSource" "type" should be "external" +And their "ruleSource" "isCustomized" should depend on whether the rule was customized +``` + +#### Scenario: Exporting both prebuilt and custom rules in bulk +```Gherkin +Given a space with prebuilt and custom rules installed +When the user selects prebuilt rules in the alerts table +And chooses "Export" from bulk actions +Then the selected rules should be exported as an NDJSON file +And the prebuilt rules should include an "immutable" field with a value of true +And the custom rules should include an "immutable" field with a value of false +And the prebuilt rules' "ruleSource" "type" should be "external" +And the custom rules' "ruleSource" "type" should be "internal" +``` + +### Error Handling + +#### Scenario: Exporting beyond the export limit +```Gherkin +Given a space with prebuilt and custom rules installed +And the number of rules is greater than the export limit (defaults to 10_000) +Then the request should be rejected as a bad request +``` diff --git a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/importing.md b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/importing.md new file mode 100644 index 0000000000000..0c947d0a52b95 --- /dev/null +++ b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/importing.md @@ -0,0 +1,127 @@ +# Prebuilt Rule Import + +This is a test plan for the importing of prebuilt rules. This feature is an aspect of `Milestone 2` of the [Rule Immutability/Customization](https://github.com/elastic/security-team/issues/1974) epic. + +Status: `in progress`. + +## Useful information + +### Tickets + +- [Rule Immutability/Customization](https://github.com/elastic/security-team/issues/1974) +- [Rule Importing Feature](https://github.com/elastic/kibana/issues/180168) +- [Rule Import API PR](https://github.com/elastic/kibana/pull/190198) + +### Terminology + +- **prebuilt rule**: A rule contained in our `Prebuilt Security Detection Rules` integration in Fleet. +- **custom rule**: A rule defined by the user, which has no relation to the prebuilt rules +- **rule source, or ruleSource**: A field on the rule that defines the rule's categorization + +## Scenarios + +### Core Functionality + +#### Scenario: Importing an unmodified prebuilt rule with a matching rule_id and version + +```Gherkin +Given the import payload contains a prebuilt rule with a matching rule_id and version, identical to the published rule +When the user imports the rule +Then the rule should be created or updated +And the ruleSource type should be "external" +And isCustomized should be false +``` + +#### Scenario: Importing a customized prebuilt rule with a matching rule_id and version + +```Gherkin +Given the import payload contains a prebuilt rule with a matching rule_id and version, modified from the published version +When the user imports the rule +Then the rule should be created or updated +And the ruleSource type should be "external" +And isCustomized should be true +``` + +#### Scenario: Importing a prebuilt rule with a matching rule_id but no matching version + +```Gherkin +Given the import payload contains a prebuilt rule with a matching rule_id but no matching version +When the user imports the rule +Then the rule should be created or updated +And the ruleSource type should be "external" +And isCustomized should be true +``` + +#### Scenario: Importing a prebuilt rule with a non-existent rule_id + +```Gherkin +Given the import payload contains a prebuilt rule with a non-existent rule_id +When the user imports the rule +Then the rule should be created +And the ruleSource type should be "internal" +``` + +#### Scenario: Importing a prebuilt rule without a rule_id field + +```Gherkin +Given the import payload contains a prebuilt rule without a rule_id field +When the user imports the rule +Then the import should be rejected with a message "rule_id field is required" +``` + +#### Scenario: Importing a prebuilt rule with a matching rule_id but missing a version field + +```Gherkin +Given the import payload contains a prebuilt rule without a version field +When the user imports the rule +Then the import should be rejected with a message "version field is required" +``` + +#### Scenario: Importing an existing custom rule missing a version field + +```Gherkin +Given the import payload contains an existing custom rule without a version field +When the user imports the rule +Then the rule should be updated +And the ruleSource type should be "internal" +And the "version" field should be set to the existing rule's "version" +``` + +#### Scenario: Importing a new custom rule missing a version field + +```Gherkin +Given the import payload contains a new custom rule without a version field +When the user imports the rule +Then the rule should be created +And the ruleSource type should be "internal" +And the "version" field should be set to 1 +``` + +#### Scenario: Importing a rule with overwrite flag set to true + +```Gherkin +Given the import payload contains a rule with an existing rule_id +And the overwrite flag is set to true +When the user imports the rule +Then the rule should be overwritten +And the ruleSource type should be calculated based on the rule_id and version +``` + +#### Scenario: Importing a rule with overwrite flag set to false + +```Gherkin +Given the import payload contains a rule with an existing rule_id +And the overwrite flag is set to false +When the user imports the rule +Then the import should be rejected with a message "rule_id already exists" +``` + +#### Scenario: Importing both custom and prebuilt rules + +```Gherkin +Given the import payload contains modified and unmodified, custom and prebuilt rules +When the user imports the rule +Then custom rules should be created or updated, with versions defaulted to 1 +And prebuilt rules should be created or updated, +And prebuilt rules missing versions should be rejected +``` diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx index cc8f2abda9c4e..9bcf35fdb13ca 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.test.tsx @@ -202,7 +202,9 @@ const onOpenTimeline = jest.fn(); const COMBO_BOX_TOGGLE_BUTTON_TEST_ID = 'comboBoxToggleListButton'; const VERSION_INPUT_TEST_ID = 'relatedIntegrationVersionDependency'; -describe('StepDefineRule', () => { +// Failing: See https://github.com/elastic/kibana/issues/199648 +// Failing: See https://github.com/elastic/kibana/issues/199700 +describe.skip('StepDefineRule', () => { beforeEach(() => { jest.clearAllMocks(); mockUseRuleFromTimeline.mockReturnValue({ onOpenTimeline, loading: false }); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx index be65593364593..af9e8dca1f24f 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx @@ -32,7 +32,6 @@ export const GraphPreviewContainer: React.FC = () => { const graphFetchQuery = useFetchGraphData({ req: { query: { - actorIds: [], eventIds, start: DEFAULT_FROM, end: DEFAULT_TO, diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx new file mode 100644 index 0000000000000..c22ec0caa82c5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { useFetchGraphData } from './use_fetch_graph_data'; + +const mockUseQuery = jest.fn(); + +jest.mock('@tanstack/react-query', () => { + return { + useQuery: (...args: unknown[]) => mockUseQuery(...args), + }; +}); + +describe('useFetchGraphData', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('Should pass default options when options are not provided', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: true, + refetchOnWindowFocus: true, + }); + }); + + it('Should should not be enabled when enabled set to false', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + options: { + enabled: false, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: false, + refetchOnWindowFocus: true, + }); + }); + + it('Should should not be refetchOnWindowFocus when refetchOnWindowFocus set to false', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + options: { + refetchOnWindowFocus: false, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: true, + refetchOnWindowFocus: false, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts index 2304cfb8d4fd2..9a0e270a9b2e0 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts @@ -10,6 +10,7 @@ import type { GraphRequest, GraphResponse, } from '@kbn/cloud-security-posture-common/types/graph/latest'; +import { useMemo } from 'react'; import { EVENT_GRAPH_VISUALIZATION_API } from '../../../../../common/constants'; import { useHttp } from '../../../../common/lib/kibana'; @@ -30,6 +31,11 @@ export interface UseFetchGraphDataParams { * Defaults to true. */ enabled?: boolean; + /** + * If true, the query will refetch on window focus. + * Defaults to true. + */ + refetchOnWindowFocus?: boolean; }; } @@ -61,18 +67,25 @@ export const useFetchGraphData = ({ req, options, }: UseFetchGraphDataParams): UseFetchGraphDataResult => { - const { actorIds, eventIds, start, end } = req.query; + const { eventIds, start, end, esQuery } = req.query; const http = useHttp(); + const QUERY_KEY = useMemo( + () => ['useFetchGraphData', eventIds, start, end, esQuery], + [end, esQuery, eventIds, start] + ); const { isLoading, isError, data } = useQuery( - ['useFetchGraphData', actorIds, eventIds, start, end], + QUERY_KEY, () => { return http.post(EVENT_GRAPH_VISUALIZATION_API, { version: '1', body: JSON.stringify(req), }); }, - options + { + enabled: options?.enabled ?? true, + refetchOnWindowFocus: options?.refetchOnWindowFocus ?? true, + } ); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/enrich_signal_threat_matches.ts index 8f98eab1a93e9..0d9882fe8aec3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/enrich_signal_threat_matches.ts @@ -24,8 +24,10 @@ export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): Signa if (existingSignalHit == null) { acc[signalId] = signalHit; } else { - const existingQueries = existingSignalHit?.matched_queries ?? []; - const newQueries = signalHit.matched_queries ?? []; + const existingQueries = Array.isArray(existingSignalHit?.matched_queries) + ? existingSignalHit.matched_queries + : []; + const newQueries = Array.isArray(signalHit.matched_queries) ? signalHit.matched_queries : []; existingSignalHit.matched_queries = [...existingQueries, ...newQueries]; acc[signalId] = existingSignalHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_signals_map_from_threat_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_signals_map_from_threat_index.ts index 309516a57335c..9694d37aab0ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_signals_map_from_threat_index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_signals_map_from_threat_index.ts @@ -90,7 +90,9 @@ export async function getSignalsQueryMapFromThreatIndex( while (maxThreatsReachedMap.size < eventsCount && threatList?.hits.hits.length > 0) { threatList.hits.hits.forEach((threatHit) => { - const matchedQueries = threatHit?.matched_queries || []; + const matchedQueries = Array.isArray(threatHit?.matched_queries) + ? threatHit.matched_queries + : []; matchedQueries.forEach((matchedQuery) => { const decodedQuery = decodeThreatMatchNamedQuery(matchedQuery); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts index da72d121c371c..347ea5d1d94c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/utils.ts @@ -189,7 +189,9 @@ export const decodeThreatMatchNamedQuery = (encoded: string): DecodedThreatNamed export const extractNamedQueries = ( hit: SignalSourceHit | ThreatListItem ): DecodedThreatNamedQuery[] => - hit.matched_queries?.map((match) => decodeThreatMatchNamedQuery(match)) ?? []; + Array.isArray(hit.matched_queries) + ? hit.matched_queries.map((match) => decodeThreatMatchNamedQuery(match)) + : []; export const buildExecutionIntervalValidator: (interval: string) => () => void = (interval) => { const intervalDuration = parseInterval(interval); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts index e15aceb8a713b..54af298d11a3e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts @@ -46,16 +46,19 @@ export const buildIndicatorShouldClauses = ( export const buildIndicatorEnrichments = (hits: estypes.SearchHit[]): CtiEnrichment[] => { return hits.flatMap(({ matched_queries: matchedQueries, ...hit }) => { return ( - matchedQueries?.reduce((enrichments, matchedQuery) => { - if (isValidEventField(matchedQuery)) { - enrichments.push({ - ...hit.fields, - ...buildIndicatorMatchedFields(hit, matchedQuery), - }); - } + (Array.isArray(matchedQueries) ? matchedQueries : [])?.reduce( + (enrichments, matchedQuery) => { + if (isValidEventField(matchedQuery)) { + enrichments.push({ + ...hit.fields, + ...buildIndicatorMatchedFields(hit, matchedQuery), + }); + } - return enrichments; - }, []) ?? [] + return enrichments; + }, + [] + ) ?? [] ); }); }; diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 45960195be216..cd820d1e70780 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -29,7 +29,7 @@ import { TaskManagerConfig } from './config'; import { createInitialMiddleware, addMiddlewareToChain, Middleware } from './lib/middleware'; import { removeIfExists } from './lib/remove_if_exists'; import { setupSavedObjects, BACKGROUND_TASK_NODE_SO_NAME, TASK_SO_NAME } from './saved_objects'; -import { TaskDefinitionRegistry, TaskTypeDictionary, REMOVED_TYPES } from './task_type_dictionary'; +import { TaskDefinitionRegistry, TaskTypeDictionary } from './task_type_dictionary'; import { AggregationOpts, FetchResult, SearchOpts, TaskStore } from './task_store'; import { createManagedConfiguration } from './lib/create_managed_configuration'; import { TaskScheduling } from './task_scheduling'; @@ -45,6 +45,10 @@ import { metricsStream, Metrics } from './metrics'; import { TaskManagerMetricsCollector } from './metrics/task_metrics_collector'; import { TaskPartitioner } from './lib/task_partitioner'; import { getDefaultCapacity } from './lib/get_default_capacity'; +import { + registerMarkRemovedTasksAsUnrecognizedDefinition, + scheduleMarkRemovedTasksAsUnrecognizedDefinition, +} from './removed_tasks/mark_removed_tasks_as_unrecognized'; export interface TaskManagerSetupContract { /** @@ -221,6 +225,11 @@ export class TaskManagerPlugin } registerDeleteInactiveNodesTaskDefinition(this.logger, core.getStartServices, this.definitions); + registerMarkRemovedTasksAsUnrecognizedDefinition( + this.logger, + core.getStartServices, + this.definitions + ); if (this.config.unsafe.exclude_task_types.length) { this.logger.warn( @@ -332,7 +341,6 @@ export class TaskManagerPlugin this.taskPollingLifecycle = new TaskPollingLifecycle({ config: this.config!, definitions: this.definitions, - unusedTypes: REMOVED_TYPES, logger: this.logger, executionContext, taskStore, @@ -384,6 +392,7 @@ export class TaskManagerPlugin }); scheduleDeleteInactiveNodesTaskDefinition(this.logger, taskScheduling).catch(() => {}); + scheduleMarkRemovedTasksAsUnrecognizedDefinition(this.logger, taskScheduling).catch(() => {}); return { fetch: (opts: SearchOpts): Promise => taskStore.fetch(opts), diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index 1f244f7f4c8a5..a408bd3f634d9 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -106,7 +106,6 @@ describe('TaskPollingLifecycle', () => { }, taskStore: mockTaskStore, logger: taskManagerLogger, - unusedTypes: [], definitions: new TaskTypeDictionary(taskManagerLogger), middleware: createInitialMiddleware(), startingCapacity: 20, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index 0b1710ae7fa2f..fb6776fa34f28 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -55,7 +55,6 @@ export interface ITaskEventEmitter { export type TaskPollingLifecycleOpts = { logger: Logger; definitions: TaskTypeDictionary; - unusedTypes: string[]; taskStore: TaskStore; config: TaskManagerConfig; middleware: Middleware; @@ -115,7 +114,6 @@ export class TaskPollingLifecycle implements ITaskEventEmitter this.pool.availableCapacity(taskType), taskPartitioner, diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts index 76df8b7ae5584..fa1d1f749985b 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts @@ -70,7 +70,6 @@ describe('mark_available_tasks_as_claimed', () => { fieldUpdates, claimableTaskTypes: definitions.getAllTypes(), skippedTaskTypes: [], - unusedTaskTypes: [], taskMaxAttempts: Array.from(definitions).reduce((accumulator, [type, { maxAttempts }]) => { return { ...accumulator, [type]: maxAttempts || defaultMaxAttempts }; }, {}), @@ -153,8 +152,6 @@ if (doc['task.runAt'].size()!=0) { ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) .join(' ')} - } else if (params.unusedTaskTypes.contains(ctx._source.task.taskType)) { - ctx._source.task.status = "unrecognized"; } else { ctx.op = "noop"; }`, @@ -167,7 +164,6 @@ if (doc['task.runAt'].size()!=0) { }, claimableTaskTypes: ['sampleTask', 'otherTask'], skippedTaskTypes: [], - unusedTaskTypes: [], taskMaxAttempts: { sampleTask: 5, otherTask: 1, @@ -242,7 +238,6 @@ if (doc['task.runAt'].size()!=0) { fieldUpdates, claimableTaskTypes: ['foo', 'bar'], skippedTaskTypes: [], - unusedTaskTypes: [], taskMaxAttempts: { foo: 5, bar: 2, diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts index 4e138545aec25..ec99c6ad5bf80 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts @@ -202,7 +202,6 @@ export interface UpdateFieldsAndMarkAsFailedOpts { }; claimableTaskTypes: string[]; skippedTaskTypes: string[]; - unusedTaskTypes: string[]; taskMaxAttempts: { [field: string]: number }; } @@ -210,7 +209,6 @@ export const updateFieldsAndMarkAsFailed = ({ fieldUpdates, claimableTaskTypes, skippedTaskTypes, - unusedTaskTypes, taskMaxAttempts, }: UpdateFieldsAndMarkAsFailedOpts): ScriptClause => { const setScheduledAtScript = `if(ctx._source.task.retryAt != null && ZonedDateTime.parse(ctx._source.task.retryAt).toInstant().toEpochMilli() < params.now) { @@ -227,8 +225,6 @@ export const updateFieldsAndMarkAsFailed = ({ source: ` if (params.claimableTaskTypes.contains(ctx._source.task.taskType)) { ${setScheduledAtAndMarkAsClaimed} - } else if (params.unusedTaskTypes.contains(ctx._source.task.taskType)) { - ctx._source.task.status = "unrecognized"; } else { ctx.op = "noop"; }`, @@ -238,7 +234,6 @@ export const updateFieldsAndMarkAsFailed = ({ fieldUpdates, claimableTaskTypes, skippedTaskTypes, - unusedTaskTypes, taskMaxAttempts, }, }; diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts index 437af8e007bdb..629e3464399c7 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.test.ts @@ -83,7 +83,6 @@ describe('TaskClaiming', () => { strategy: 'non-default', definitions, excludedTaskTypes: [], - unusedTypes: [], taskStore: taskStoreMock.create({ taskManagerId: '' }), maxAttempts: 2, getAvailableCapacity: () => 10, @@ -134,7 +133,6 @@ describe('TaskClaiming', () => { strategy: 'default', definitions, excludedTaskTypes: [], - unusedTypes: [], taskStore: taskStoreMock.create({ taskManagerId: '' }), maxAttempts: 2, getAvailableCapacity: () => 10, diff --git a/x-pack/plugins/task_manager/server/queries/task_claiming.ts b/x-pack/plugins/task_manager/server/queries/task_claiming.ts index c9bca31755408..1b1e414903628 100644 --- a/x-pack/plugins/task_manager/server/queries/task_claiming.ts +++ b/x-pack/plugins/task_manager/server/queries/task_claiming.ts @@ -34,7 +34,6 @@ export interface TaskClaimingOpts { logger: Logger; strategy: string; definitions: TaskTypeDictionary; - unusedTypes: string[]; taskStore: TaskStore; maxAttempts: number; excludedTaskTypes: string[]; @@ -92,7 +91,6 @@ export class TaskClaiming { private readonly taskClaimingBatchesByType: TaskClaimingBatches; private readonly taskMaxAttempts: Record; private readonly excludedTaskTypes: string[]; - private readonly unusedTypes: string[]; private readonly taskClaimer: TaskClaimerFn; private readonly taskPartitioner: TaskPartitioner; @@ -111,7 +109,6 @@ export class TaskClaiming { this.taskClaimingBatchesByType = this.partitionIntoClaimingBatches(this.definitions); this.taskMaxAttempts = Object.fromEntries(this.normalizeMaxAttempts(this.definitions)); this.excludedTaskTypes = opts.excludedTaskTypes; - this.unusedTypes = opts.unusedTypes; this.taskClaimer = getTaskClaimer(this.logger, opts.strategy); this.events$ = new Subject(); this.taskPartitioner = opts.taskPartitioner; @@ -178,7 +175,6 @@ export class TaskClaiming { taskStore: this.taskStore, events$: this.events$, getCapacity: this.getAvailableCapacity, - unusedTypes: this.unusedTypes, definitions: this.definitions, taskMaxAttempts: this.taskMaxAttempts, excludedTaskTypes: this.excludedTaskTypes, diff --git a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.test.ts b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.test.ts new file mode 100644 index 0000000000000..1485216a67f33 --- /dev/null +++ b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.test.ts @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockLogger } from '../test_utils'; +import { coreMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { SCHEDULE_INTERVAL, taskRunner } from './mark_removed_tasks_as_unrecognized'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +const createTaskDoc = (id: string = '1'): SearchHit => ({ + _index: '.kibana_task_manager_9.0.0_001', + _id: `task:${id}`, + _score: 1, + _source: { + references: [], + type: 'task', + updated_at: '2024-11-06T14:17:55.935Z', + task: { + taskType: 'report', + params: '{}', + state: '{"foo":"test"}', + stateVersion: 1, + runAt: '2024-11-06T14:17:55.935Z', + enabled: true, + scheduledAt: '2024-11-06T14:17:55.935Z', + attempts: 0, + status: 'idle', + startedAt: null, + retryAt: null, + ownerId: null, + partition: 211, + }, + }, +}); + +describe('markRemovedTasksAsUnrecognizedTask', () => { + const logger = mockLogger(); + const coreSetup = coreMock.createSetup(); + const esClient = elasticsearchServiceMock.createStart(); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('marks removed tasks as unrecognized', async () => { + esClient.client.asInternalUser.bulk.mockResolvedValue({ + errors: false, + took: 0, + items: [ + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:123', + _version: 2, + result: 'updated', + _shards: { total: 1, successful: 1, failed: 0 }, + _seq_no: 84, + _primary_term: 1, + status: 200, + }, + }, + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:456', + _version: 2, + result: 'updated', + _shards: { total: 1, successful: 1, failed: 0 }, + _seq_no: 84, + _primary_term: 1, + status: 200, + }, + }, + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:789', + _version: 2, + result: 'updated', + _shards: { total: 1, successful: 1, failed: 0 }, + _seq_no: 84, + _primary_term: 1, + status: 200, + }, + }, + ], + }); + + coreSetup.getStartServices.mockResolvedValue([ + { + ...coreMock.createStart(), + elasticsearch: esClient, + }, + {}, + coreMock.createSetup(), + ]); + // @ts-expect-error + esClient.client.asInternalUser.search.mockResponse({ + hits: { hits: [createTaskDoc('123'), createTaskDoc('456'), createTaskDoc('789')], total: 3 }, + }); + + const runner = taskRunner(logger, coreSetup.getStartServices)(); + const result = await runner.run(); + + expect(esClient.client.asInternalUser.bulk).toHaveBeenCalledWith({ + body: [ + { update: { _id: 'task:123' } }, + { doc: { task: { status: 'unrecognized' } } }, + { update: { _id: 'task:456' } }, + { doc: { task: { status: 'unrecognized' } } }, + { update: { _id: 'task:789' } }, + { doc: { task: { status: 'unrecognized' } } }, + ], + index: '.kibana_task_manager', + refresh: false, + }); + + expect(logger.debug).toHaveBeenCalledWith(`Marked 3 removed tasks as unrecognized`); + + expect(result).toEqual({ + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }); + }); + + it('skips update when there are no removed task types', async () => { + coreSetup.getStartServices.mockResolvedValue([ + { + ...coreMock.createStart(), + elasticsearch: esClient, + }, + {}, + coreMock.createSetup(), + ]); + // @ts-expect-error + esClient.client.asInternalUser.search.mockResponse({ + hits: { hits: [], total: 0 }, + }); + + const runner = taskRunner(logger, coreSetup.getStartServices)(); + const result = await runner.run(); + + expect(esClient.client.asInternalUser.bulk).not.toHaveBeenCalled(); + + expect(result).toEqual({ + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }); + }); + + it('schedules the next run even when there is an error', async () => { + coreSetup.getStartServices.mockResolvedValue([ + { + ...coreMock.createStart(), + elasticsearch: esClient, + }, + {}, + coreMock.createSetup(), + ]); + esClient.client.asInternalUser.search.mockRejectedValueOnce(new Error('foo')); + + const runner = taskRunner(logger, coreSetup.getStartServices)(); + const result = await runner.run(); + + expect(esClient.client.asInternalUser.bulk).not.toHaveBeenCalled(); + + expect(logger.error).toHaveBeenCalledWith( + 'Failed to mark removed tasks as unrecognized. Error: foo' + ); + + expect(result).toEqual({ + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }); + }); + + it('handles partial errors from bulk partial update', async () => { + esClient.client.asInternalUser.bulk.mockResolvedValue({ + errors: false, + took: 0, + items: [ + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:123', + _version: 2, + result: 'updated', + _shards: { total: 1, successful: 1, failed: 0 }, + _seq_no: 84, + _primary_term: 1, + status: 200, + }, + }, + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:456', + _version: 2, + result: 'updated', + _shards: { total: 1, successful: 1, failed: 0 }, + _seq_no: 84, + _primary_term: 1, + status: 200, + }, + }, + { + update: { + _index: '.kibana_task_manager_9.0.0_001', + _id: 'task:789', + _version: 2, + error: { + type: 'document_missing_exception', + reason: '[5]: document missing', + index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', + shard: '0', + index: '.kibana_task_manager_9.0.0_001', + }, + status: 404, + }, + }, + ], + }); + + coreSetup.getStartServices.mockResolvedValue([ + { + ...coreMock.createStart(), + elasticsearch: esClient, + }, + {}, + coreMock.createSetup(), + ]); + // @ts-expect-error + esClient.client.asInternalUser.search.mockResponse({ + hits: { hits: [createTaskDoc('123'), createTaskDoc('456'), createTaskDoc('789')], total: 3 }, + }); + + const runner = taskRunner(logger, coreSetup.getStartServices)(); + const result = await runner.run(); + + expect(esClient.client.asInternalUser.bulk).toHaveBeenCalledWith({ + body: [ + { update: { _id: 'task:123' } }, + { doc: { task: { status: 'unrecognized' } } }, + { update: { _id: 'task:456' } }, + { doc: { task: { status: 'unrecognized' } } }, + { update: { _id: 'task:789' } }, + { doc: { task: { status: 'unrecognized' } } }, + ], + index: '.kibana_task_manager', + refresh: false, + }); + expect(logger.warn).toHaveBeenCalledWith( + `Error updating task task:789 to mark as unrecognized - {\"type\":\"document_missing_exception\",\"reason\":\"[5]: document missing\",\"index_uuid\":\"aAsFqTI0Tc2W0LCWgPNrOA\",\"shard\":\"0\",\"index\":\".kibana_task_manager_9.0.0_001\"}` + ); + + expect(logger.debug).toHaveBeenCalledWith(`Marked 2 removed tasks as unrecognized`); + + expect(result).toEqual({ + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts new file mode 100644 index 0000000000000..e28d5221e72d5 --- /dev/null +++ b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/logging'; +import { CoreStart } from '@kbn/core-lifecycle-server'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { TaskScheduling } from '../task_scheduling'; +import { TaskTypeDictionary } from '../task_type_dictionary'; +import { ConcreteTaskInstance, TaskManagerStartContract } from '..'; +import { TaskStatus } from '../task'; +import { REMOVED_TYPES } from '../task_type_dictionary'; +import { TASK_MANAGER_INDEX } from '../constants'; + +export const TASK_ID = 'mark_removed_tasks_as_unrecognized'; +const TASK_TYPE = `task_manager:${TASK_ID}`; + +export const SCHEDULE_INTERVAL = '1h'; + +export async function scheduleMarkRemovedTasksAsUnrecognizedDefinition( + logger: Logger, + taskScheduling: TaskScheduling +) { + try { + await taskScheduling.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + schedule: { interval: SCHEDULE_INTERVAL }, + state: {}, + params: {}, + }); + } catch (e) { + logger.error(`Error scheduling ${TASK_ID} task, received ${e.message}`); + } +} + +export function registerMarkRemovedTasksAsUnrecognizedDefinition( + logger: Logger, + coreStartServices: () => Promise<[CoreStart, TaskManagerStartContract, unknown]>, + taskTypeDictionary: TaskTypeDictionary +) { + taskTypeDictionary.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Mark removed tasks as unrecognized', + createTaskRunner: taskRunner(logger, coreStartServices), + }, + }); +} + +export function taskRunner( + logger: Logger, + coreStartServices: () => Promise<[CoreStart, TaskManagerStartContract, unknown]> +) { + return () => { + return { + async run() { + try { + const [{ elasticsearch }] = await coreStartServices(); + const esClient = elasticsearch.client.asInternalUser; + + const removedTasks = await queryForRemovedTasks(esClient); + + if (removedTasks.length > 0) { + await updateTasksToBeUnrecognized(esClient, logger, removedTasks); + } + + return { + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }; + } catch (e) { + logger.error(`Failed to mark removed tasks as unrecognized. Error: ${e.message}`); + return { + state: {}, + schedule: { interval: SCHEDULE_INTERVAL }, + }; + } + }, + }; + }; +} + +async function queryForRemovedTasks( + esClient: ElasticsearchClient +): Promise>> { + const result = await esClient.search({ + index: TASK_MANAGER_INDEX, + body: { + size: 100, + _source: false, + query: { + bool: { + must: [ + { + terms: { + 'task.taskType': REMOVED_TYPES, + }, + }, + ], + }, + }, + }, + }); + + return result.hits.hits; +} + +async function updateTasksToBeUnrecognized( + esClient: ElasticsearchClient, + logger: Logger, + removedTasks: Array> +) { + const bulkBody = []; + for (const task of removedTasks) { + bulkBody.push({ update: { _id: task._id } }); + bulkBody.push({ doc: { task: { status: TaskStatus.Unrecognized } } }); + } + + let removedCount = 0; + try { + const removeResults = await esClient.bulk({ + index: TASK_MANAGER_INDEX, + refresh: false, + body: bulkBody, + }); + for (const removeResult of removeResults.items) { + if (!removeResult.update || !removeResult.update._id) { + logger.warn( + `Error updating task with unknown to mark as unrecognized - malformed response` + ); + } else if (removeResult.update?.error) { + logger.warn( + `Error updating task ${ + removeResult.update._id + } to mark as unrecognized - ${JSON.stringify(removeResult.update.error)}` + ); + } else { + removedCount++; + } + } + logger.debug(`Marked ${removedCount} removed tasks as unrecognized`); + } catch (err) { + // don't worry too much about errors, we'll try again next time + logger.warn(`Error updating tasks to mark as unrecognized: ${err}`); + } +} diff --git a/x-pack/plugins/task_manager/server/task_claimers/index.ts b/x-pack/plugins/task_manager/server/task_claimers/index.ts index 178ebacf68cb9..f41c489fd7550 100644 --- a/x-pack/plugins/task_manager/server/task_claimers/index.ts +++ b/x-pack/plugins/task_manager/server/task_claimers/index.ts @@ -26,7 +26,6 @@ export interface TaskClaimerOpts { events$: Subject; taskStore: TaskStore; definitions: TaskTypeDictionary; - unusedTypes: string[]; excludedTaskTypes: string[]; taskMaxAttempts: Record; logger: Logger; diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.test.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.test.ts index fe44ce9e94c68..07dae3c48a392 100644 --- a/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.test.ts +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.test.ts @@ -190,7 +190,6 @@ describe('TaskClaiming', () => { definitions, taskStore: store, excludedTaskTypes, - unusedTypes: unusedTaskTypes, maxAttempts: taskClaimingOpts.maxAttempts ?? 2, getAvailableCapacity: taskClaimingOpts.getAvailableCapacity ?? (() => 10), taskPartitioner, @@ -206,20 +205,17 @@ describe('TaskClaiming', () => { claimingOpts, hits = [generateFakeTasks(1)], excludedTaskTypes = [], - unusedTaskTypes = [], }: { storeOpts: Partial; taskClaimingOpts: Partial; claimingOpts: Omit; hits?: ConcreteTaskInstance[][]; excludedTaskTypes?: string[]; - unusedTaskTypes?: string[]; }) { const { taskClaiming, store } = initialiseTestClaiming({ storeOpts, taskClaimingOpts, excludedTaskTypes, - unusedTaskTypes, hits, }); @@ -355,7 +351,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -378,7 +373,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 3; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 3; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); @@ -440,312 +435,6 @@ describe('TaskClaiming', () => { expect(result.docs.length).toEqual(3); }); - test('should not claim tasks of removed type', async () => { - const store = taskStoreMock.create({ taskManagerId: 'test-test' }); - store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); - - const fetchedTasks = [ - mockInstance({ id: `id-1`, taskType: 'report' }), - mockInstance({ id: `id-2`, taskType: 'report' }), - mockInstance({ id: `id-3`, taskType: 'yawn' }), - ]; - - const { versionMap, docLatestVersions } = getVersionMapsFromTasks(fetchedTasks); - store.msearch.mockResolvedValueOnce({ docs: fetchedTasks, versionMap }); - store.getDocVersions.mockResolvedValueOnce(docLatestVersions); - - store.bulkGet.mockResolvedValueOnce([fetchedTasks[2]].map(asOk)); - store.bulkPartialUpdate.mockResolvedValueOnce([fetchedTasks[2]].map(getPartialUpdateResult)); - store.bulkPartialUpdate.mockResolvedValueOnce( - [fetchedTasks[0], fetchedTasks[1]].map(getPartialUpdateResult) - ); - - const taskClaiming = new TaskClaiming({ - logger: taskManagerLogger, - strategy: CLAIM_STRATEGY_MGET, - definitions: taskDefinitions, - taskStore: store, - excludedTaskTypes: [], - unusedTypes: ['report'], - maxAttempts: 2, - getAvailableCapacity: () => 10, - taskPartitioner, - }); - - const resultOrErr = await taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ - claimOwnershipUntil: new Date(), - }); - - if (!isOk(resultOrErr)) { - expect(resultOrErr).toBe(undefined); - } - - const result = unwrap(resultOrErr) as ClaimOwnershipResult; - - expect(apm.startTransaction).toHaveBeenCalledWith( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - expect(mockApmTrans.end).toHaveBeenCalledWith('success'); - - expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 1; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 2;', - { tags: ['taskClaiming', 'claimAvailableTasksMget'] } - ); - - expect(store.msearch.mock.calls[0][0]?.[0]).toMatchObject({ - size: 40, - seq_no_primary_term: true, - }); - expect(store.getDocVersions).toHaveBeenCalledWith(['task:id-1', 'task:id-2', 'task:id-3']); - expect(store.bulkPartialUpdate).toHaveBeenCalledTimes(2); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(1, [ - { - id: fetchedTasks[2].id, - version: fetchedTasks[2].version, - scheduledAt: fetchedTasks[2].runAt, - attempts: 1, - ownerId: 'test-test', - retryAt: new Date('1970-01-01T00:05:30.000Z'), - status: 'running', - startedAt: new Date('1970-01-01T00:00:00.000Z'), - }, - ]); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(2, [ - { - id: fetchedTasks[0].id, - version: fetchedTasks[0].version, - status: 'unrecognized', - }, - { - id: fetchedTasks[1].id, - version: fetchedTasks[1].version, - status: 'unrecognized', - }, - ]); - expect(store.bulkGet).toHaveBeenCalledWith(['id-3']); - - expect(result.stats).toEqual({ - tasksClaimed: 1, - tasksConflicted: 0, - tasksErrors: 0, - tasksUpdated: 1, - tasksLeftUnclaimed: 0, - staleTasks: 0, - }); - expect(result.docs.length).toEqual(1); - }); - - test('should log warning if error updating single removed task as unrecognized', async () => { - const store = taskStoreMock.create({ taskManagerId: 'test-test' }); - store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); - - const fetchedTasks = [ - mockInstance({ id: `id-1`, taskType: 'report' }), - mockInstance({ id: `id-2`, taskType: 'report' }), - mockInstance({ id: `id-3`, taskType: 'yawn' }), - ]; - - const { versionMap, docLatestVersions } = getVersionMapsFromTasks(fetchedTasks); - store.msearch.mockResolvedValueOnce({ docs: fetchedTasks, versionMap }); - store.getDocVersions.mockResolvedValueOnce(docLatestVersions); - - store.bulkGet.mockResolvedValueOnce([fetchedTasks[2]].map(asOk)); - store.bulkPartialUpdate.mockResolvedValueOnce([fetchedTasks[2]].map(getPartialUpdateResult)); - store.bulkPartialUpdate.mockResolvedValueOnce([ - asOk(fetchedTasks[0]), - asErr({ - type: 'task', - id: fetchedTasks[1].id, - status: 404, - error: { - type: 'document_missing_exception', - reason: '[5]: document missing', - index_uuid: 'aAsFqTI0Tc2W0LCWgPNrOA', - shard: '0', - index: '.kibana_task_manager_8.16.0_001', - }, - }), - ]); - - const taskClaiming = new TaskClaiming({ - logger: taskManagerLogger, - strategy: CLAIM_STRATEGY_MGET, - definitions: taskDefinitions, - taskStore: store, - excludedTaskTypes: [], - unusedTypes: ['report'], - maxAttempts: 2, - getAvailableCapacity: () => 10, - taskPartitioner, - }); - - const resultOrErr = await taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ - claimOwnershipUntil: new Date(), - }); - - if (!isOk(resultOrErr)) { - expect(resultOrErr).toBe(undefined); - } - - const result = unwrap(resultOrErr) as ClaimOwnershipResult; - - expect(apm.startTransaction).toHaveBeenCalledWith( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - expect(mockApmTrans.end).toHaveBeenCalledWith('success'); - - expect(taskManagerLogger.warn).toHaveBeenCalledWith( - 'Error updating task id-2:task to mark as unrecognized during claim: {"type":"document_missing_exception","reason":"[5]: document missing","index_uuid":"aAsFqTI0Tc2W0LCWgPNrOA","shard":"0","index":".kibana_task_manager_8.16.0_001"}', - { tags: ['taskClaiming', 'claimAvailableTasksMget'] } - ); - expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 1; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 1;', - { tags: ['taskClaiming', 'claimAvailableTasksMget'] } - ); - - expect(store.msearch.mock.calls[0][0]?.[0]).toMatchObject({ - size: 40, - seq_no_primary_term: true, - }); - expect(store.getDocVersions).toHaveBeenCalledWith(['task:id-1', 'task:id-2', 'task:id-3']); - expect(store.bulkPartialUpdate).toHaveBeenCalledTimes(2); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(1, [ - { - id: fetchedTasks[2].id, - version: fetchedTasks[2].version, - scheduledAt: fetchedTasks[2].runAt, - attempts: 1, - ownerId: 'test-test', - retryAt: new Date('1970-01-01T00:05:30.000Z'), - status: 'running', - startedAt: new Date('1970-01-01T00:00:00.000Z'), - }, - ]); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(2, [ - { - id: fetchedTasks[0].id, - version: fetchedTasks[0].version, - status: 'unrecognized', - }, - { - id: fetchedTasks[1].id, - version: fetchedTasks[1].version, - status: 'unrecognized', - }, - ]); - expect(store.bulkGet).toHaveBeenCalledWith(['id-3']); - - expect(result.stats).toEqual({ - tasksClaimed: 1, - tasksConflicted: 0, - tasksErrors: 0, - tasksUpdated: 1, - tasksLeftUnclaimed: 0, - staleTasks: 0, - }); - expect(result.docs.length).toEqual(1); - }); - - test('should log warning if error updating all removed tasks as unrecognized', async () => { - const store = taskStoreMock.create({ taskManagerId: 'test-test' }); - store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); - - const fetchedTasks = [ - mockInstance({ id: `id-1`, taskType: 'report' }), - mockInstance({ id: `id-2`, taskType: 'report' }), - mockInstance({ id: `id-3`, taskType: 'yawn' }), - ]; - - const { versionMap, docLatestVersions } = getVersionMapsFromTasks(fetchedTasks); - store.msearch.mockResolvedValueOnce({ docs: fetchedTasks, versionMap }); - store.getDocVersions.mockResolvedValueOnce(docLatestVersions); - - store.bulkGet.mockResolvedValueOnce([fetchedTasks[2]].map(asOk)); - store.bulkPartialUpdate.mockResolvedValueOnce([fetchedTasks[2]].map(getPartialUpdateResult)); - store.bulkPartialUpdate.mockRejectedValueOnce(new Error('Oh no')); - - const taskClaiming = new TaskClaiming({ - logger: taskManagerLogger, - strategy: CLAIM_STRATEGY_MGET, - definitions: taskDefinitions, - taskStore: store, - excludedTaskTypes: [], - unusedTypes: ['report'], - maxAttempts: 2, - getAvailableCapacity: () => 10, - taskPartitioner, - }); - - const resultOrErr = await taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ - claimOwnershipUntil: new Date(), - }); - - if (!isOk(resultOrErr)) { - expect(resultOrErr).toBe(undefined); - } - - const result = unwrap(resultOrErr) as ClaimOwnershipResult; - - expect(apm.startTransaction).toHaveBeenCalledWith( - TASK_MANAGER_MARK_AS_CLAIMED, - TASK_MANAGER_TRANSACTION_TYPE - ); - expect(mockApmTrans.end).toHaveBeenCalledWith('success'); - - expect(taskManagerLogger.warn).toHaveBeenCalledWith( - 'Error updating tasks to mark as unrecognized during claim: Error: Oh no', - { tags: ['taskClaiming', 'claimAvailableTasksMget'] } - ); - expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 1; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', - { tags: ['taskClaiming', 'claimAvailableTasksMget'] } - ); - - expect(store.msearch.mock.calls[0][0]?.[0]).toMatchObject({ - size: 40, - seq_no_primary_term: true, - }); - expect(store.getDocVersions).toHaveBeenCalledWith(['task:id-1', 'task:id-2', 'task:id-3']); - expect(store.bulkGet).toHaveBeenCalledWith(['id-3']); - expect(store.bulkPartialUpdate).toHaveBeenCalledTimes(2); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(1, [ - { - id: fetchedTasks[2].id, - version: fetchedTasks[2].version, - scheduledAt: fetchedTasks[2].runAt, - attempts: 1, - ownerId: 'test-test', - retryAt: new Date('1970-01-01T00:05:30.000Z'), - status: 'running', - startedAt: new Date('1970-01-01T00:00:00.000Z'), - }, - ]); - expect(store.bulkPartialUpdate).toHaveBeenNthCalledWith(2, [ - { - id: fetchedTasks[0].id, - version: fetchedTasks[0].version, - status: 'unrecognized', - }, - { - id: fetchedTasks[1].id, - version: fetchedTasks[1].version, - status: 'unrecognized', - }, - ]); - - expect(result.stats).toEqual({ - tasksClaimed: 1, - tasksConflicted: 0, - tasksErrors: 0, - tasksUpdated: 1, - tasksLeftUnclaimed: 0, - staleTasks: 0, - }); - expect(result.docs.length).toEqual(1); - }); - test('should handle no tasks to claim', async () => { const store = taskStoreMock.create({ taskManagerId: 'test-test' }); store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); @@ -761,7 +450,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -828,7 +516,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -851,7 +538,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 2; stale: 0; conflicts: 0; missing: 1; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 2; stale: 0; conflicts: 0; missing: 1; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); @@ -922,7 +609,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -945,7 +631,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 2; stale: 0; conflicts: 0; missing: 1; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 2; stale: 0; conflicts: 0; missing: 1; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); @@ -1016,7 +702,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1039,7 +724,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 2; stale: 1; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 2; stale: 1; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); @@ -1116,7 +801,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1139,7 +823,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 4; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 4; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); @@ -1248,7 +932,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1271,7 +954,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 1; removed: 0;', + 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 1;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); expect(taskManagerLogger.error).toHaveBeenCalledWith( @@ -1377,7 +1060,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1400,7 +1082,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 3; stale: 0; conflicts: 1; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 3; stale: 0; conflicts: 1; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); expect(taskManagerLogger.warn).toHaveBeenCalledWith( @@ -1504,7 +1186,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1619,7 +1300,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1642,7 +1322,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 1; getErrors: 0; removed: 0;', + 'task claimer claimed: 3; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 1; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); expect(taskManagerLogger.error).toHaveBeenCalledWith( @@ -1753,7 +1433,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -1776,7 +1455,7 @@ describe('TaskClaiming', () => { expect(mockApmTrans.end).toHaveBeenCalledWith('success'); expect(taskManagerLogger.debug).toHaveBeenCalledWith( - 'task claimer claimed: 3; stale: 0; conflicts: 1; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0; removed: 0;', + 'task claimer claimed: 3; stale: 0; conflicts: 1; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0;', { tags: ['taskClaiming', 'claimAvailableTasksMget'] } ); expect(taskManagerLogger.error).not.toHaveBeenCalled(); @@ -1870,7 +1549,6 @@ describe('TaskClaiming', () => { definitions: taskDefinitions, taskStore: store, excludedTaskTypes: [], - unusedTypes: [], maxAttempts: 2, getAvailableCapacity: () => 10, taskPartitioner, @@ -2488,7 +2166,6 @@ describe('TaskClaiming', () => { strategy: CLAIM_STRATEGY_MGET, definitions, excludedTaskTypes: [], - unusedTypes: [], taskStore, maxAttempts: 2, getAvailableCapacity, diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.ts index 16d9ba5c7fae7..431daab8dd2cb 100644 --- a/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.ts +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_mget.ts @@ -57,7 +57,6 @@ interface OwnershipClaimingOpts { claimOwnershipUntil: Date; size: number; taskTypes: Set; - removedTypes: Set; getCapacity: (taskType?: string | undefined) => number; excludedTaskTypePatterns: string[]; taskStore: TaskStore; @@ -90,19 +89,16 @@ export async function claimAvailableTasksMget( async function claimAvailableTasks(opts: TaskClaimerOpts): Promise { const { getCapacity, claimOwnershipUntil, batches, events$, taskStore, taskPartitioner } = opts; - const { definitions, unusedTypes, excludedTaskTypes, taskMaxAttempts } = opts; + const { definitions, excludedTaskTypes, taskMaxAttempts } = opts; const logger = createWrappedLogger({ logger: opts.logger, tags: [claimAvailableTasksMget.name] }); const initialCapacity = getCapacity(); const stopTaskTimer = startTaskTimer(); - const removedTypes = new Set(unusedTypes); // REMOVED_TYPES - // get a list of candidate tasks to claim, with their version info const { docs, versionMap } = await searchAvailableTasks({ definitions, taskTypes: new Set(definitions.getAllTypes()), excludedTaskTypePatterns: excludedTaskTypes, - removedTypes, taskStore, events$, claimOwnershipUntil, @@ -125,18 +121,12 @@ async function claimAvailableTasks(opts: TaskClaimerOpts): Promise `task:${doc.id}`)); - // filter out stale, missing and removed tasks + // filter out stale and missing tasks const currentTasks: ConcreteTaskInstance[] = []; const staleTasks: ConcreteTaskInstance[] = []; const missingTasks: ConcreteTaskInstance[] = []; - const removedTasks: ConcreteTaskInstance[] = []; for (const searchDoc of docs) { - if (removedTypes.has(searchDoc.taskType)) { - removedTasks.push(searchDoc); - continue; - } - const searchVersion = versionMap.get(searchDoc.id); const latestVersion = docLatestVersions.get(`task:${searchDoc.id}`); if (!searchVersion || !latestVersion) { @@ -236,42 +226,8 @@ async function claimAvailableTasks(opts: TaskClaimerOpts): Promise 0) { - const tasksToRemove = Array.from(removedTasks); - const tasksToRemoveUpdates: PartialConcreteTaskInstance[] = []; - for (const task of tasksToRemove) { - tasksToRemoveUpdates.push({ - id: task.id, - status: TaskStatus.Unrecognized, - }); - } - - // don't worry too much about errors, we'll get them next time - try { - const removeResults = await taskStore.bulkPartialUpdate(tasksToRemoveUpdates); - for (const removeResult of removeResults) { - if (isOk(removeResult)) { - removedCount++; - } else { - const { id, type, error } = removeResult.error; - logger.warn( - `Error updating task ${id}:${type} to mark as unrecognized during claim: ${JSON.stringify( - error - )}` - ); - } - } - } catch (err) { - // swallow the error because this is unrelated to the claim cycle - logger.warn(`Error updating tasks to mark as unrecognized during claim: ${err}`); - } - } - // TODO: need a better way to generate stats - const message = `task claimer claimed: ${fullTasksToRun.length}; stale: ${staleTasks.length}; conflicts: ${conflicts}; missing: ${missingTasks.length}; capacity reached: ${leftOverTasks.length}; updateErrors: ${bulkUpdateErrors}; getErrors: ${bulkGetErrors}; removed: ${removedCount};`; + const message = `task claimer claimed: ${fullTasksToRun.length}; stale: ${staleTasks.length}; conflicts: ${conflicts}; missing: ${missingTasks.length}; capacity reached: ${leftOverTasks.length}; updateErrors: ${bulkUpdateErrors}; getErrors: ${bulkGetErrors};`; logger.debug(message); // build results @@ -306,7 +262,6 @@ export const NO_ASSIGNED_PARTITIONS_WARNING_INTERVAL = 60000; async function searchAvailableTasks({ definitions, taskTypes, - removedTypes, excludedTaskTypePatterns, taskStore, getCapacity, @@ -318,7 +273,6 @@ async function searchAvailableTasks({ const claimPartitions = buildClaimPartitions({ types: taskTypes, excludedTaskTypes, - removedTypes, getCapacity, definitions, }); @@ -352,10 +306,7 @@ async function searchAvailableTasks({ // Task must be enabled EnabledTask, // a task type that's not excluded (may be removed or not) - OneOfTaskTypes( - 'task.taskType', - claimPartitions.unlimitedTypes.concat(Array.from(removedTypes)) - ), + OneOfTaskTypes('task.taskType', claimPartitions.unlimitedTypes), // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), @@ -407,7 +358,6 @@ async function searchAvailableTasks({ } interface ClaimPartitions { - removedTypes: string[]; unlimitedTypes: string[]; limitedTypes: Map; } @@ -415,30 +365,23 @@ interface ClaimPartitions { interface BuildClaimPartitionsOpts { types: Set; excludedTaskTypes: Set; - removedTypes: Set; getCapacity: (taskType?: string) => number; definitions: TaskTypeDictionary; } function buildClaimPartitions(opts: BuildClaimPartitionsOpts): ClaimPartitions { const result: ClaimPartitions = { - removedTypes: [], unlimitedTypes: [], limitedTypes: new Map(), }; - const { types, excludedTaskTypes, removedTypes, getCapacity, definitions } = opts; + const { types, excludedTaskTypes, getCapacity, definitions } = opts; for (const type of types) { const definition = definitions.get(type); if (definition == null) continue; if (excludedTaskTypes.has(type)) continue; - if (removedTypes.has(type)) { - result.removedTypes.push(type); - continue; - } - if (definition.maxConcurrency == null) { result.unlimitedTypes.push(definition.type); continue; diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.test.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.test.ts index 13e6faf2de0fd..623693e71c54d 100644 --- a/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.test.ts +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.test.ts @@ -99,14 +99,12 @@ describe('TaskClaiming', () => { hits = [generateFakeTasks(1)], versionConflicts = 2, excludedTaskTypes = [], - unusedTaskTypes = [], }: { storeOpts: Partial; taskClaimingOpts: Partial; hits?: ConcreteTaskInstance[][]; versionConflicts?: number; excludedTaskTypes?: string[]; - unusedTaskTypes?: string[]; }) { const definitions = storeOpts.definitions ?? taskDefinitions; const store = taskStoreMock.create({ taskManagerId: storeOpts.taskManagerId }); @@ -136,7 +134,6 @@ describe('TaskClaiming', () => { definitions, taskStore: store, excludedTaskTypes, - unusedTypes: unusedTaskTypes, maxAttempts: taskClaimingOpts.maxAttempts ?? 2, getAvailableCapacity: taskClaimingOpts.getAvailableCapacity ?? (() => 10), taskPartitioner, @@ -153,7 +150,6 @@ describe('TaskClaiming', () => { hits = [generateFakeTasks(1)], versionConflicts = 2, excludedTaskTypes = [], - unusedTaskTypes = [], }: { storeOpts: Partial; taskClaimingOpts: Partial; @@ -161,14 +157,12 @@ describe('TaskClaiming', () => { hits?: ConcreteTaskInstance[][]; versionConflicts?: number; excludedTaskTypes?: string[]; - unusedTaskTypes?: string[]; }) { const getCapacity = taskClaimingOpts.getAvailableCapacity ?? (() => 10); const { taskClaiming, store } = initialiseTestClaiming({ storeOpts, taskClaimingOpts, excludedTaskTypes, - unusedTaskTypes, hits, versionConflicts, }); @@ -471,7 +465,6 @@ if (doc['task.runAt'].size()!=0) { 'anotherLimitedToOne', 'limitedToTwo', ], - unusedTaskTypes: [], taskMaxAttempts: { unlimited: maxAttempts, }, @@ -493,7 +486,6 @@ if (doc['task.runAt'].size()!=0) { 'anotherLimitedToOne', 'limitedToTwo', ], - unusedTaskTypes: [], taskMaxAttempts: { limitedToOne: maxAttempts, }, @@ -640,7 +632,6 @@ if (doc['task.runAt'].size()!=0) { }, taskPartitioner, excludedTaskTypes: [], - unusedTypes: [], }); const resultOrErr = await taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ @@ -848,129 +839,6 @@ if (doc['task.runAt'].size()!=0) { expect(firstCycle).not.toMatchObject(secondCycle); }); - test('it passes any unusedTaskTypes to script', async () => { - const maxAttempts = _.random(2, 43); - const customMaxAttempts = _.random(44, 100); - const taskManagerId = uuidv1(); - const fieldUpdates = { - ownerId: taskManagerId, - retryAt: new Date(Date.now()), - }; - const definitions = new TaskTypeDictionary(mockLogger()); - definitions.registerTaskDefinitions({ - foo: { - title: 'foo', - createTaskRunner: jest.fn(), - }, - bar: { - title: 'bar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - foobar: { - title: 'foobar', - maxAttempts: customMaxAttempts, - createTaskRunner: jest.fn(), - }, - }); - - const { - args: { - updateByQuery: [{ query, script }], - }, - } = await testClaimAvailableTasks({ - storeOpts: { - definitions, - taskManagerId, - }, - taskClaimingOpts: { - maxAttempts, - }, - claimingOpts: { - claimOwnershipUntil: new Date(), - }, - excludedTaskTypes: ['foobar'], - unusedTaskTypes: ['barfoo'], - }); - expect(query).toMatchObject({ - bool: { - must: [ - { - bool: { - must: [ - { - term: { - 'task.enabled': true, - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - must: [ - { term: { 'task.status': 'idle' } }, - { range: { 'task.runAt': { lte: 'now' } } }, - ], - }, - }, - { - bool: { - must: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - }, - }, - { range: { 'task.retryAt': { lte: 'now' } } }, - ], - }, - }, - ], - }, - }, - ], - filter: [ - { - bool: { - must_not: [ - { - bool: { - should: [ - { term: { 'task.status': 'running' } }, - { term: { 'task.status': 'claiming' } }, - ], - must: { range: { 'task.retryAt': { gt: 'now' } } }, - }, - }, - ], - }, - }, - ], - }, - }); - expect(script).toMatchObject({ - source: expect.any(String), - lang: 'painless', - params: { - fieldUpdates, - claimableTaskTypes: ['foo', 'bar'], - skippedTaskTypes: ['foobar'], - unusedTaskTypes: ['barfoo'], - taskMaxAttempts: { - bar: customMaxAttempts, - foo: maxAttempts, - }, - }, - }); - }); - test('it claims tasks by setting their ownerId, status and retryAt', async () => { const taskManagerId = uuidv1(); const claimOwnershipUntil = new Date(Date.now()); @@ -1356,7 +1224,6 @@ if (doc['task.runAt'].size()!=0) { strategy: 'update_by_query', definitions, excludedTaskTypes: [], - unusedTypes: [], taskStore, maxAttempts: 2, getAvailableCapacity, diff --git a/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.ts b/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.ts index 5a4bccb43b984..fdfd09e07f9c7 100644 --- a/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.ts +++ b/x-pack/plugins/task_manager/server/task_claimers/strategy_update_by_query.ts @@ -51,7 +51,6 @@ interface OwnershipClaimingOpts { taskStore: TaskStore; events$: Subject; definitions: TaskTypeDictionary; - unusedTypes: string[]; excludedTaskTypes: string[]; taskMaxAttempts: Record; } @@ -60,7 +59,7 @@ export async function claimAvailableTasksUpdateByQuery( opts: TaskClaimerOpts ): Promise { const { getCapacity, claimOwnershipUntil, batches, events$, taskStore } = opts; - const { definitions, unusedTypes, excludedTaskTypes, taskMaxAttempts } = opts; + const { definitions, excludedTaskTypes, taskMaxAttempts } = opts; const initialCapacity = getCapacity(); let accumulatedResult = getEmptyClaimOwnershipResult(); @@ -83,7 +82,6 @@ export async function claimAvailableTasksUpdateByQuery( taskTypes: isLimited(batch) ? new Set([batch.tasksTypes]) : batch.tasksTypes, taskStore, definitions, - unusedTypes, excludedTaskTypes, taskMaxAttempts, }); @@ -137,7 +135,6 @@ async function markAvailableTasksAsClaimed({ claimOwnershipUntil, size, taskTypes, - unusedTypes, taskMaxAttempts, }: OwnershipClaimingOpts): Promise { const { taskTypesToSkip = [], taskTypesToClaim = [] } = groupBy( @@ -164,7 +161,6 @@ async function markAvailableTasksAsClaimed({ }, claimableTaskTypes: taskTypesToClaim, skippedTaskTypes: taskTypesToSkip, - unusedTaskTypes: unusedTypes, taskMaxAttempts: pick(taskMaxAttempts, taskTypesToClaim), }); diff --git a/x-pack/plugins/task_manager/tsconfig.json b/x-pack/plugins/task_manager/tsconfig.json index 55ad764c5bdcd..b11eaaf44a905 100644 --- a/x-pack/plugins/task_manager/tsconfig.json +++ b/x-pack/plugins/task_manager/tsconfig.json @@ -27,7 +27,8 @@ "@kbn/logging", "@kbn/core-lifecycle-server", "@kbn/cloud-plugin", - "@kbn/core-saved-objects-base-server-internal" + "@kbn/core-saved-objects-base-server-internal", + "@kbn/core-elasticsearch-server", ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 9d89589574213..f03c509001511 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -862,11 +862,7 @@ "controls.controlGroup.manageControl.dataSource.dataViewTitle": "Vue de données", "controls.controlGroup.manageControl.dataSource.fieldListErrorTitle": "Erreur lors du chargement de la liste des champs", "controls.controlGroup.manageControl.dataSource.fieldTitle": "Champ", - "controls.controlGroup.manageControl.dataSource.formGroupDescription": "Sélectionnez la vue de données et le champ pour lesquels vous voulez créer un contrôle.", - "controls.controlGroup.manageControl.dataSource.formGroupTitle": "Source de données", "controls.controlGroup.manageControl.dataSource.selectDataViewMessage": "Veuillez sélectionner une vue de données", - "controls.controlGroup.manageControl.displaySettings.formGroupDescription": "Changez la manière dont le contrôle apparaît sur votre tableau de bord.", - "controls.controlGroup.manageControl.displaySettings.formGroupTitle": "Paramètres d'affichage", "controls.controlGroup.manageControl.displaySettings.growSwitchTitle": "Augmenter la largeur en fonction de l'espace disponible", "controls.controlGroup.manageControl.displaySettings.titleInputTitle": "Étiquette", "controls.controlGroup.manageControl.displaySettings.widthInputTitle": "Largeur minimale", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index c923e7c19a858..0b5587b2bd916 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -864,11 +864,7 @@ "controls.controlGroup.manageControl.dataSource.dataViewTitle": "データビュー", "controls.controlGroup.manageControl.dataSource.fieldListErrorTitle": "フィールドリストの読み込みエラー", "controls.controlGroup.manageControl.dataSource.fieldTitle": "フィールド", - "controls.controlGroup.manageControl.dataSource.formGroupDescription": "コントロールを作成するデータビューとフィールドを選択します。", - "controls.controlGroup.manageControl.dataSource.formGroupTitle": "データソース", "controls.controlGroup.manageControl.dataSource.selectDataViewMessage": "データビューを選択してください", - "controls.controlGroup.manageControl.displaySettings.formGroupDescription": "ダッシュボードにコントロールを表示する方法を変更します。", - "controls.controlGroup.manageControl.displaySettings.formGroupTitle": "表示設定", "controls.controlGroup.manageControl.displaySettings.growSwitchTitle": "空きスペースに合わせて幅を拡大", "controls.controlGroup.manageControl.displaySettings.titleInputTitle": "ラベル", "controls.controlGroup.manageControl.displaySettings.widthInputTitle": "最小幅", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index ad65daedc7406..4d4db174396e2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -856,11 +856,7 @@ "controls.controlGroup.manageControl.dataSource.dataViewTitle": "数据视图", "controls.controlGroup.manageControl.dataSource.fieldListErrorTitle": "加载字段列表时出错", "controls.controlGroup.manageControl.dataSource.fieldTitle": "字段", - "controls.controlGroup.manageControl.dataSource.formGroupDescription": "选择要为其创建控件的数据视图和字段。", - "controls.controlGroup.manageControl.dataSource.formGroupTitle": "数据源", "controls.controlGroup.manageControl.dataSource.selectDataViewMessage": "请选择数据视图", - "controls.controlGroup.manageControl.displaySettings.formGroupDescription": "更改控件在仪表板上的显示方式。", - "controls.controlGroup.manageControl.displaySettings.formGroupTitle": "显示设置", "controls.controlGroup.manageControl.displaySettings.growSwitchTitle": "扩大宽度以适应可用空间", "controls.controlGroup.manageControl.displaySettings.titleInputTitle": "标签", "controls.controlGroup.manageControl.displaySettings.widthInputTitle": "最小宽度", diff --git a/x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/server/plugin.ts b/x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/server/plugin.ts index 6e4eba065ec70..f6e3383270436 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/server/plugin.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/server/plugin.ts @@ -95,6 +95,34 @@ export class SampleTaskManagerFixturePlugin return res.ok({ body: {} }); } ); + + router.post( + { + path: '/api/alerting_tasks/run_mark_tasks_as_unrecognized', + validate: { + body: schema.object({}), + }, + }, + async ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise> => { + try { + const taskManager = await this.taskManagerStart; + await taskManager.ensureScheduled({ + id: 'mark_removed_tasks_as_unrecognized', + taskType: 'task_manager:mark_removed_tasks_as_unrecognized', + schedule: { interval: '1h' }, + state: {}, + params: {}, + }); + return res.ok({ body: await taskManager.runSoon('mark_removed_tasks_as_unrecognized') }); + } catch (err) { + return res.ok({ body: { id: 'mark_removed_tasks_as_unrecognized', error: `${err}` } }); + } + } + ); } public start(core: CoreStart, { taskManager }: SampleTaskManagerFixtureStartDeps) { diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/scheduled_task_id.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/scheduled_task_id.ts index 0086dd2679c74..f05075be810a1 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/scheduled_task_id.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/scheduled_task_id.ts @@ -138,6 +138,12 @@ export default function createScheduledTaskIdTests({ getService }: FtrProviderCo // When we enable the rule, the unrecognized task should be removed and a new // task created in its place + await supertestWithoutAuth + .post('/api/alerting_tasks/run_mark_tasks_as_unrecognized') + .set('kbn-xsrf', 'foo') + .send({}) + .expect(200); + // scheduled task should exist and be unrecognized await retry.try(async () => { const taskRecordLoaded = await getScheduledTask(RULE_ID); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/constants/archiver.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/constants/archiver.ts new file mode 100644 index 0000000000000..6afc2e9eca63b --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/constants/archiver.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +type ArchiveName = + | '8.0.0' + | 'apm_8.0.0' + | 'apm_mappings_only_8.0.0' + | 'infra_metrics_and_apm' + | 'metrics_8.0.0' + | 'ml_8.0.0' + | 'observability_overview' + | 'rum_8.0.0' + | 'rum_test_data'; + +export const ARCHIVER_ROUTES: { [key in ArchiveName]: string } = { + '8.0.0': 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0', + 'apm_8.0.0': 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_8.0.0', + 'apm_mappings_only_8.0.0': + 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_mappings_only_8.0.0', + infra_metrics_and_apm: + 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/infra_metrics_and_apm', + 'metrics_8.0.0': 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/metrics_8.0.0', + 'ml_8.0.0': 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/ml_8.0.0', + observability_overview: + 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/observability_overview', + 'rum_8.0.0': 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/rum_8.0.0', + rum_test_data: 'x-pack/test/apm_api_integration/common/fixtures/es_archiver/rum_test_data', +}; diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/failed_transactions.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/failed_transactions.spec.ts new file mode 100644 index 0000000000000..549f48009197f --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/failed_transactions.spec.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { orderBy } from 'lodash'; +import expect from '@kbn/expect'; +import type { FailedTransactionsCorrelationsResponse } from '@kbn/apm-plugin/common/correlations/failed_transactions_correlations/types'; +import { EVENT_OUTCOME } from '@kbn/apm-plugin/common/es_fields/apm'; +import { EventOutcome } from '@kbn/apm-plugin/common/event_outcome'; +import { LatencyDistributionChartType } from '@kbn/apm-plugin/common/latency_distribution_chart_types'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; + +// These tests go through the full sequence of queries required +// to get the final results for a failed transactions correlation analysis. +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); + // This matches the parameters used for the other tab's queries in `../correlations/*`. + const getOptions = () => ({ + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + }); + + describe('failed transactions', () => { + describe('without data', () => { + it('handles the empty state', async () => { + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + chartType: LatencyDistributionChartType.failedTransactionsCorrelations, + }, + }, + }); + + expect(overallDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${overallDistributionResponse.status}'` + ); + + const errorDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], + chartType: LatencyDistributionChartType.failedTransactionsCorrelations, + }, + }, + }); + + expect(errorDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${errorDistributionResponse.status}'` + ); + + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates/transactions', + params: { + query: getOptions(), + }, + }); + + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` + ); + + const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/p_values/transactions', + params: { + body: { + ...getOptions(), + fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, + }, + }, + }); + + expect(failedTransactionsCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` + ); + + const finalRawResponse: FailedTransactionsCorrelationsResponse = { + ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + failedTransactionsCorrelations: + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, + }; + + expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( + 0, + `Expected 0 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` + ); + }); + }); + + describe('with data', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); + + it('runs queries and returns results', async () => { + const overallDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + chartType: LatencyDistributionChartType.failedTransactionsCorrelations, + }, + }, + }); + + expect(overallDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${overallDistributionResponse.status}'` + ); + + const errorDistributionResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', + params: { + body: { + ...getOptions(), + percentileThreshold: 95, + termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], + chartType: LatencyDistributionChartType.failedTransactionsCorrelations, + }, + }, + }); + + expect(errorDistributionResponse.status).to.eql( + 200, + `Expected status to be '200', got '${errorDistributionResponse.status}'` + ); + + const fieldCandidatesResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/correlations/field_candidates/transactions', + params: { + query: getOptions(), + }, + }); + + expect(fieldCandidatesResponse.status).to.eql( + 200, + `Expected status to be '200', got '${fieldCandidatesResponse.status}'` + ); + + const fieldCandidates = fieldCandidatesResponse.body?.fieldCandidates.filter( + (t) => !(t === EVENT_OUTCOME) + ); + + // Identified 80 fieldCandidates. + expect(fieldCandidates.length).to.eql( + 80, + `Expected field candidates length to be '80', got '${fieldCandidates.length}'` + ); + + const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ + endpoint: 'POST /internal/apm/correlations/p_values/transactions', + params: { + body: { + ...getOptions(), + fieldCandidates, + }, + }, + }); + + expect(failedTransactionsCorrelationsResponse.status).to.eql( + 200, + `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` + ); + + const fieldsToSample = new Set(); + if ( + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.length > 0 + ) { + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.forEach( + (d) => { + fieldsToSample.add(d.fieldName); + } + ); + } + + const finalRawResponse: FailedTransactionsCorrelationsResponse = { + ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, + percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, + overallHistogram: overallDistributionResponse.body?.overallHistogram, + errorHistogram: errorDistributionResponse.body?.overallHistogram, + failedTransactionsCorrelations: + failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, + }; + + expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); + expect(finalRawResponse?.errorHistogram?.length).to.be(101); + expect(finalRawResponse?.overallHistogram?.length).to.be(101); + + expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( + 29, + `Expected 29 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` + ); + + const sortedCorrelations = orderBy( + finalRawResponse?.failedTransactionsCorrelations, + ['score', 'fieldName', 'fieldValue'], + ['desc', 'asc', 'asc'] + ); + const correlation = sortedCorrelations?.[0]; + + expect(typeof correlation).to.be('object'); + expect(correlation?.doc_count).to.be(31); + expect(correlation?.score).to.be(83.70467673605746); + expect(correlation?.bg_count).to.be(31); + expect(correlation?.fieldName).to.be('transaction.result'); + expect(correlation?.fieldValue).to.be('HTTP 5xx'); + expect(typeof correlation?.pValue).to.be('number'); + expect(typeof correlation?.normalizedScore).to.be('number'); + expect(typeof correlation?.failurePercentage).to.be('number'); + expect(typeof correlation?.successPercentage).to.be('number'); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_candidates.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_candidates.spec.ts new file mode 100644 index 0000000000000..8db9a7df05fd3 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_candidates.spec.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); + + const endpoint = 'GET /internal/apm/correlations/field_candidates/transactions'; + + const getOptions = () => ({ + params: { + query: { + environment: 'ENVIRONMENT_ALL', + start: '2020', + end: '2021', + kuery: '', + }, + }, + }); + + describe('field candidates', () => { + describe('without data', () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.be(200); + // If the source indices are empty, there will be no field candidates + // because of the `include_empty_fields: false` option in the query. + expect(response.body?.fieldCandidates.length).to.be(0); + }); + }); + + describe('with data and default args', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); + + it('returns field candidates', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); + + expect(response.status).to.eql(200); + expect(response.body?.fieldCandidates.length).to.be(81); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_value_pairs.spec.ts similarity index 59% rename from x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_value_pairs.spec.ts index 4765e83342e52..9fcd438421b6a 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/field_value_pairs.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/field_value_pairs.spec.ts @@ -6,11 +6,12 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); const endpoint = 'POST /internal/apm/correlations/field_value_pairs/transactions'; @@ -41,22 +42,27 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); - registry.when('field value pairs without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint, - ...getOptions(), - }); + describe('field value pairs', () => { + describe('without data', () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); - expect(response.status).to.be(200); - expect(response.body?.fieldValuePairs.length).to.be(0); + expect(response.status).to.be(200); + expect(response.body?.fieldValuePairs.length).to.be(0); + }); }); - }); - registry.when( - 'field value pairs with data and default args', - { config: 'trial', archives: ['8.0.0'] }, - () => { + describe('with data and default args', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); + it('returns field value pairs', async () => { const response = await apmApiClient.readUser({ endpoint, @@ -66,6 +72,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.eql(200); expect(response.body?.fieldValuePairs.length).to.be(124); }); - } - ); + }); + }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/index.ts new file mode 100644 index 0000000000000..660556edb8d79 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { + describe('correlations', () => { + loadTestFile(require.resolve('./failed_transactions.spec.ts')); + loadTestFile(require.resolve('./field_candidates.spec.ts')); + loadTestFile(require.resolve('./field_value_pairs.spec.ts')); + loadTestFile(require.resolve('./latency.spec.ts')); + loadTestFile(require.resolve('./p_values.spec.ts')); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/latency.spec.ts similarity index 93% rename from x-pack/test/apm_api_integration/tests/correlations/latency.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/latency.spec.ts index 5326136976428..e0080806f6a0e 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/latency.spec.ts @@ -14,13 +14,14 @@ import type { LatencyCorrelationsResponse, } from '@kbn/apm-plugin/common/correlations/latency_correlations/types'; import { LatencyDistributionChartType } from '@kbn/apm-plugin/common/latency_distribution_chart_types'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; // These tests go through the full sequence of queries required // to get the final results for a latency correlation analysis. -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); // This matches the parameters used for the other tab's queries in `../correlations/*`. const getOptions = () => ({ @@ -30,10 +31,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { kuery: '', }); - registry.when( - 'correlations latency overall without data', - { config: 'trial', archives: [] }, - () => { + describe('latency', () => { + describe('overall without data', () => { it('handles the empty state', async () => { const overallDistributionResponse = await apmApiClient.readUser({ endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', @@ -104,13 +103,16 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(finalRawResponse?.overallHistogram).to.be(undefined); expect(finalRawResponse?.latencyCorrelations?.length).to.be(0); }); - } - ); + }); + + describe('with data and opbeans-node args', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); - registry.when( - 'correlations latency with data and opbeans-node args', - { config: 'trial', archives: ['8.0.0'] }, - () => { // putting this into a single `it` because the responses depend on each other it('runs queries and returns results', async () => { const overallDistributionResponse = await apmApiClient.readUser({ @@ -250,6 +252,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(correlation?.ksTest).to.be(1.9848961005439386e-12); expect(correlation?.histogram?.length).to.be(101); }); - } - ); + }); + }); } diff --git a/x-pack/test/apm_api_integration/tests/correlations/p_values.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/p_values.spec.ts similarity index 58% rename from x-pack/test/apm_api_integration/tests/correlations/p_values.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/p_values.spec.ts index 42a9fdadbb480..ba6e3384cec93 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/p_values.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/p_values.spec.ts @@ -6,11 +6,12 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); const endpoint = 'POST /internal/apm/correlations/p_values/transactions'; @@ -41,22 +42,27 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); - registry.when('p values without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint, - ...getOptions(), - }); + describe('p values', () => { + describe('without data', () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); - expect(response.status).to.be(200); - expect(response.body?.failedTransactionsCorrelations.length).to.be(0); + expect(response.status).to.be(200); + expect(response.body?.failedTransactionsCorrelations.length).to.be(0); + }); }); - }); - registry.when( - 'p values with data and default args', - { config: 'trial', archives: ['8.0.0'] }, - () => { + describe('with data and default args', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); + it('returns p values', async () => { const response = await apmApiClient.readUser({ endpoint, @@ -66,6 +72,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.eql(200); expect(response.body?.failedTransactionsCorrelations.length).to.be(15); }); - } - ); + }); + }); } diff --git a/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/significant_correlations.spec.ts similarity index 71% rename from x-pack/test/apm_api_integration/tests/correlations/significant_correlations.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/significant_correlations.spec.ts index d4450c192a029..e1f968d178868 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/significant_correlations.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/correlations/significant_correlations.spec.ts @@ -6,11 +6,12 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); const endpoint = 'POST /internal/apm/correlations/significant_correlations/transactions'; @@ -65,22 +66,27 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); - registry.when('significant correlations without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint, - ...getOptions(), - }); + describe('significant correlations', () => { + describe('without data', () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint, + ...getOptions(), + }); - expect(response.status).to.be(200); - expect(response.body?.latencyCorrelations.length).to.be(0); + expect(response.status).to.be(200); + expect(response.body?.latencyCorrelations.length).to.be(0); + }); }); - }); - registry.when( - 'significant correlations with data and default args', - { config: 'trial', archives: ['8.0.0'] }, - () => { + describe('with data and default args', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); + it('returns significant correlations', async () => { const response = await apmApiClient.readUser({ endpoint, @@ -90,6 +96,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.eql(200); expect(response.body?.latencyCorrelations.length).to.be(7); }); - } - ); + }); + }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts index f8c0352984473..f1e8fc381a072 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts @@ -15,6 +15,7 @@ export default function apmApiIntegrationTests({ loadTestFile(require.resolve('./mobile')); loadTestFile(require.resolve('./custom_dashboards')); loadTestFile(require.resolve('./dependencies')); + loadTestFile(require.resolve('./correlations')); loadTestFile(require.resolve('./entities')); loadTestFile(require.resolve('./cold_start')); }); diff --git a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json index a64e037343bb3..c79a4c6b52309 100644 --- a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json +++ b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json @@ -3786,10 +3786,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-error" - }, "mapping": { "total_fields": { "limit": "2000" @@ -4243,8 +4239,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } @@ -8183,10 +8178,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-metric" - }, "mapping": { "total_fields": { "limit": "2000" @@ -8640,8 +8631,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } @@ -12871,8 +12861,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } @@ -16653,10 +16642,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-profile" - }, "mapping": { "total_fields": { "limit": "2000" @@ -17110,8 +17095,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } @@ -20899,10 +20883,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-span" - }, "mapping": { "total_fields": { "limit": "2000" @@ -21356,8 +21336,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } @@ -25242,10 +25221,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-transaction" - }, "mapping": { "total_fields": { "limit": "2000" @@ -25699,8 +25674,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } diff --git a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.spec.ts b/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.spec.ts deleted file mode 100644 index 13754f6c7eb5a..0000000000000 --- a/x-pack/test/apm_api_integration/tests/correlations/failed_transactions.spec.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { orderBy } from 'lodash'; -import expect from '@kbn/expect'; -import type { FailedTransactionsCorrelationsResponse } from '@kbn/apm-plugin/common/correlations/failed_transactions_correlations/types'; -import { EVENT_OUTCOME } from '@kbn/apm-plugin/common/es_fields/apm'; -import { EventOutcome } from '@kbn/apm-plugin/common/event_outcome'; -import { LatencyDistributionChartType } from '@kbn/apm-plugin/common/latency_distribution_chart_types'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -// These tests go through the full sequence of queries required -// to get the final results for a failed transactions correlation analysis. -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); - - // This matches the parameters used for the other tab's queries in `../correlations/*`. - const getOptions = () => ({ - environment: 'ENVIRONMENT_ALL', - start: '2020', - end: '2021', - kuery: '', - }); - - registry.when('failed transactions without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const overallDistributionResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', - params: { - body: { - ...getOptions(), - percentileThreshold: 95, - chartType: LatencyDistributionChartType.failedTransactionsCorrelations, - }, - }, - }); - - expect(overallDistributionResponse.status).to.eql( - 200, - `Expected status to be '200', got '${overallDistributionResponse.status}'` - ); - - const errorDistributionResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', - params: { - body: { - ...getOptions(), - percentileThreshold: 95, - termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], - chartType: LatencyDistributionChartType.failedTransactionsCorrelations, - }, - }, - }); - - expect(errorDistributionResponse.status).to.eql( - 200, - `Expected status to be '200', got '${errorDistributionResponse.status}'` - ); - - const fieldCandidatesResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/correlations/field_candidates/transactions', - params: { - query: getOptions(), - }, - }); - - expect(fieldCandidatesResponse.status).to.eql( - 200, - `Expected status to be '200', got '${fieldCandidatesResponse.status}'` - ); - - const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/correlations/p_values/transactions', - params: { - body: { - ...getOptions(), - fieldCandidates: fieldCandidatesResponse.body?.fieldCandidates, - }, - }, - }); - - expect(failedTransactionsCorrelationsResponse.status).to.eql( - 200, - `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` - ); - - const finalRawResponse: FailedTransactionsCorrelationsResponse = { - ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, - percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, - overallHistogram: overallDistributionResponse.body?.overallHistogram, - failedTransactionsCorrelations: - failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, - }; - - expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( - 0, - `Expected 0 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` - ); - }); - }); - - registry.when('failed transactions with data', { config: 'trial', archives: ['8.0.0'] }, () => { - it('runs queries and returns results', async () => { - const overallDistributionResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', - params: { - body: { - ...getOptions(), - percentileThreshold: 95, - chartType: LatencyDistributionChartType.failedTransactionsCorrelations, - }, - }, - }); - - expect(overallDistributionResponse.status).to.eql( - 200, - `Expected status to be '200', got '${overallDistributionResponse.status}'` - ); - - const errorDistributionResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/latency/overall_distribution/transactions', - params: { - body: { - ...getOptions(), - percentileThreshold: 95, - termFilters: [{ fieldName: EVENT_OUTCOME, fieldValue: EventOutcome.failure }], - chartType: LatencyDistributionChartType.failedTransactionsCorrelations, - }, - }, - }); - - expect(errorDistributionResponse.status).to.eql( - 200, - `Expected status to be '200', got '${errorDistributionResponse.status}'` - ); - - const fieldCandidatesResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/correlations/field_candidates/transactions', - params: { - query: getOptions(), - }, - }); - - expect(fieldCandidatesResponse.status).to.eql( - 200, - `Expected status to be '200', got '${fieldCandidatesResponse.status}'` - ); - - const fieldCandidates = fieldCandidatesResponse.body?.fieldCandidates.filter( - (t) => !(t === EVENT_OUTCOME) - ); - - // Identified 80 fieldCandidates. - expect(fieldCandidates.length).to.eql( - 80, - `Expected field candidates length to be '80', got '${fieldCandidates.length}'` - ); - - const failedTransactionsCorrelationsResponse = await apmApiClient.readUser({ - endpoint: 'POST /internal/apm/correlations/p_values/transactions', - params: { - body: { - ...getOptions(), - fieldCandidates, - }, - }, - }); - - expect(failedTransactionsCorrelationsResponse.status).to.eql( - 200, - `Expected status to be '200', got '${failedTransactionsCorrelationsResponse.status}'` - ); - - const fieldsToSample = new Set(); - if (failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.length > 0) { - failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations.forEach((d) => { - fieldsToSample.add(d.fieldName); - }); - } - - const finalRawResponse: FailedTransactionsCorrelationsResponse = { - ccsWarning: failedTransactionsCorrelationsResponse.body?.ccsWarning, - percentileThresholdValue: overallDistributionResponse.body?.percentileThresholdValue, - overallHistogram: overallDistributionResponse.body?.overallHistogram, - errorHistogram: errorDistributionResponse.body?.overallHistogram, - failedTransactionsCorrelations: - failedTransactionsCorrelationsResponse.body?.failedTransactionsCorrelations, - }; - - expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); - expect(finalRawResponse?.errorHistogram?.length).to.be(101); - expect(finalRawResponse?.overallHistogram?.length).to.be(101); - - expect(finalRawResponse?.failedTransactionsCorrelations?.length).to.eql( - 29, - `Expected 29 identified correlations, got ${finalRawResponse?.failedTransactionsCorrelations?.length}.` - ); - - const sortedCorrelations = orderBy( - finalRawResponse?.failedTransactionsCorrelations, - ['score', 'fieldName', 'fieldValue'], - ['desc', 'asc', 'asc'] - ); - const correlation = sortedCorrelations?.[0]; - - expect(typeof correlation).to.be('object'); - expect(correlation?.doc_count).to.be(31); - expect(correlation?.score).to.be(83.70467673605746); - expect(correlation?.bg_count).to.be(31); - expect(correlation?.fieldName).to.be('transaction.result'); - expect(correlation?.fieldValue).to.be('HTTP 5xx'); - expect(typeof correlation?.pValue).to.be('number'); - expect(typeof correlation?.normalizedScore).to.be('number'); - expect(typeof correlation?.failurePercentage).to.be('number'); - expect(typeof correlation?.successPercentage).to.be('number'); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/tests/correlations/field_candidates.spec.ts b/x-pack/test/apm_api_integration/tests/correlations/field_candidates.spec.ts deleted file mode 100644 index 4a5472cf5cb23..0000000000000 --- a/x-pack/test/apm_api_integration/tests/correlations/field_candidates.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const apmApiClient = getService('apmApiClient'); - const registry = getService('registry'); - - const endpoint = 'GET /internal/apm/correlations/field_candidates/transactions'; - - const getOptions = () => ({ - params: { - query: { - environment: 'ENVIRONMENT_ALL', - start: '2020', - end: '2021', - kuery: '', - }, - }, - }); - - registry.when('field candidates without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint, - ...getOptions(), - }); - - expect(response.status).to.be(200); - // If the source indices are empty, there will be no field candidates - // because of the `include_empty_fields: false` option in the query. - expect(response.body?.fieldCandidates.length).to.be(0); - }); - }); - - registry.when( - 'field candidates with data and default args', - { config: 'trial', archives: ['8.0.0'] }, - () => { - it('returns field candidates', async () => { - const response = await apmApiClient.readUser({ - endpoint, - ...getOptions(), - }); - - expect(response.status).to.eql(200); - expect(response.body?.fieldCandidates.length).to.be(81); - }); - } - ); -} diff --git a/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json b/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json index 9f536d0bb6dc9..37f7ebdff5fb1 100644 --- a/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json +++ b/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json @@ -497,3 +497,123 @@ } } } + +{ + "type": "doc", + "value": { + "data_stream": "logs-gcp.audit-default", + "id": "5", + "index": ".ds-logs-gcp.audit-default-2024.10.07-000001", + "source": { + "@timestamp": "2024-09-01T12:34:56.789Z", + "actor": { + "entity": { + "id": "admin5@example.com" + } + }, + "client": { + "user": { + "email": "admin5@example.com" + } + }, + "cloud": { + "project": { + "id": "your-project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "google.iam.admin.v1.ListRoles", + "agent_id_status": "missing", + "category": [ + "session", + "network", + "configuration" + ], + "id": "without target", + "ingested": "2024-10-07T17:47:35Z", + "kind": "event", + "outcome": "success", + "provider": "activity", + "type": [ + "end", + "access", + "allowed" + ] + }, + "gcp": { + "audit": { + "authorization_info": [ + { + "granted": true, + "permission": "iam.roles.create", + "resource": "projects/your-project-id" + } + ], + "logentry_operation": { + "id": "operation-0987654321" + }, + "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.CreateRoleRequest", + "parent": "projects/your-project-id", + "role": { + "description": "A custom role with specific permissions", + "includedPermissions": [ + "resourcemanager.projects.get", + "resourcemanager.projects.list" + ], + "name": "projects/your-project-id/roles/customRole", + "title": "Custom Role" + }, + "roleId": "customRole" + }, + "resource_name": "projects/your-project-id/roles/customRole", + "response": { + "@type": "type.googleapis.com/google.iam.admin.v1.Role", + "description": "A custom role with specific permissions", + "includedPermissions": [ + "resourcemanager.projects.get", + "resourcemanager.projects.list" + ], + "name": "projects/your-project-id/roles/customRole", + "stage": "GA", + "title": "Custom Role" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/your-project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "ip": [ + "10.0.0.1" + ], + "user": [ + "admin3@example.com" + ] + }, + "service": { + "name": "iam.googleapis.com" + }, + "source": { + "ip": "10.0.0.1" + }, + "tags": [ + "_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-ASN.mmdb" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "google-cloud-sdk/324.0.0" + } + } + } +} diff --git a/x-pack/test/cloud_security_posture_api/routes/graph.ts b/x-pack/test/cloud_security_posture_api/routes/graph.ts index bd2f71ef3b9b2..8043e6e22feb6 100644 --- a/x-pack/test/cloud_security_posture_api/routes/graph.ts +++ b/x-pack/test/cloud_security_posture_api/routes/graph.ts @@ -11,6 +11,8 @@ import { } from '@kbn/core-http-common'; import expect from '@kbn/expect'; import type { Agent } from 'supertest'; +import { ApiMessageCode } from '@kbn/cloud-security-posture-common/types/graph/latest'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/latest'; import { FtrProviderContext } from '../ftr_provider_context'; import { result } from '../utils'; import { CspSecurityCommonProvider } from './helper/user_roles_utilites'; @@ -19,12 +21,13 @@ import { CspSecurityCommonProvider } from './helper/user_roles_utilites'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; + const logger = getService('log'); const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const cspSecurity = CspSecurityCommonProvider(providerContext); - const postGraph = (agent: Agent, body: any, auth?: { user: string; pass: string }) => { + const postGraph = (agent: Agent, body: GraphRequest, auth?: { user: string; pass: string }) => { const req = agent .post('/internal/cloud_security_posture/graph') .set(ELASTIC_HTTP_VERSION_HEADER, '1') @@ -45,7 +48,6 @@ export default function (providerContext: FtrProviderContext) { supertestWithoutAuth, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -55,19 +57,7 @@ export default function (providerContext: FtrProviderContext) { user: 'role_security_no_read_user', pass: cspSecurity.getPasswordForUser('role_security_no_read_user'), } - ).expect(result(403)); - }); - }); - - describe('Validation', () => { - it('should return 400 when missing `actorIds` field', async () => { - await postGraph(supertest, { - query: { - eventIds: [], - start: 'now-1d/d', - end: 'now/d', - }, - }).expect(result(400)); + ).expect(result(403, logger)); }); }); @@ -84,10 +74,54 @@ export default function (providerContext: FtrProviderContext) { ); }); - it('should return an empty graph', async () => { + describe('Validation', () => { + it('should return 400 when missing `eventIds` field', async () => { + await postGraph(supertest, { + // @ts-expect-error ignore error for testing + query: { + start: 'now-1d/d', + end: 'now/d', + }, + }).expect(result(400, logger)); + }); + + it('should return 400 when missing `esQuery` field is not of type bool', async () => { + await postGraph(supertest, { + query: { + eventIds: [], + start: 'now-1d/d', + end: 'now/d', + esQuery: { + // @ts-expect-error ignore error for testing + match_all: {}, + }, + }, + }).expect(result(400, logger)); + }); + + it('should return 400 with unsupported `esQuery`', async () => { + await postGraph(supertest, { + query: { + eventIds: [], + start: 'now-1d/d', + end: 'now/d', + esQuery: { + bool: { + filter: [ + { + unsupported: 'unsupported', + }, + ], + }, + }, + }, + }).expect(result(400, logger)); + }); + }); + + it('should return an empty graph / should return 200 when missing `esQuery` field', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -96,20 +130,32 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(0); expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); }); it('should return a graph with nodes and edges by actor', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -131,7 +177,6 @@ export default function (providerContext: FtrProviderContext) { it('should return a graph with nodes and edges by alert', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: ['kabcd1234efgh5678'], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', @@ -140,6 +185,7 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -161,7 +207,6 @@ export default function (providerContext: FtrProviderContext) { it('color of alert of failed event should be danger', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: ['failed-event'], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', @@ -170,6 +215,7 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -191,15 +237,26 @@ export default function (providerContext: FtrProviderContext) { it('color of event of failed event should be warning', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin2@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin2@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -219,18 +276,29 @@ export default function (providerContext: FtrProviderContext) { }); }); - it('2 grouped of events, 1 failed, 1 success', async () => { + it('2 grouped events, 1 failed, 1 success', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin3@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin3@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(5); expect(response.body).to.have.property('edges').length(6); + expect(response.body).not.to.have.property('messages'); expect(response.body.nodes[0].shape).equal('group', 'Groups should be the first nodes'); @@ -247,11 +315,167 @@ export default function (providerContext: FtrProviderContext) { response.body.edges.forEach((edge: any) => { expect(edge).to.have.property('color'); expect(edge.color).equal( - edge.id.includes('outcome(failed)') ? 'warning' : 'primary', + edge.id.includes('outcome(failed)') || + (edge.id.includes('grp(') && !edge.id.includes('outcome(success)')) + ? 'warning' + : 'primary', + `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` + ); + }); + }); + + it('should support more than 1 eventIds', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: ['kabcd1234efgh5678', 'failed-event'], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(5); + expect(response.body).to.have.property('edges').length(4); + expect(response.body).not.to.have.property('messages'); + + response.body.nodes.forEach((node: any) => { + expect(node).to.have.property('color'); + expect(node.color).equal( + 'danger', + `node color mismatched [node: ${node.id}] [actual: ${node.color}]` + ); + }); + + response.body.edges.forEach((edge: any) => { + expect(edge).to.have.property('color'); + expect(edge.color).equal( + 'danger', + `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` + ); + }); + }); + + it('should return a graph with nodes and edges by alert and actor', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: ['kabcd1234efgh5678'], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin2@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(5); + expect(response.body).to.have.property('edges').length(4); + expect(response.body).not.to.have.property('messages'); + + response.body.nodes.forEach((node: any, idx: number) => { + expect(node).to.have.property('color'); + expect(node.color).equal( + idx <= 2 // First 3 nodes are expected to be colored as danger (ORDER MATTERS, alerts are expected to be first) + ? 'danger' + : node.shape === 'label' && node.id.includes('outcome(failed)') + ? 'warning' + : 'primary', + `node color mismatched [node: ${node.id}] [actual: ${node.color}]` + ); + }); + + response.body.edges.forEach((edge: any, idx: number) => { + expect(edge).to.have.property('color'); + expect(edge.color).equal( + idx <= 1 ? 'danger' : 'warning', `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` ); }); }); + + it('Should filter unknown targets', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin5@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(0); + expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); + }); + + it('Should return unknown targets', async () => { + const response = await postGraph(supertest, { + showUnknownTarget: true, + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin5@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(3); + expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); + }); + + it('Should limit number of nodes', async () => { + const response = await postGraph(supertest, { + nodesLimit: 1, + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + exists: { + field: 'actor.entity.id', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(3); // Minimal number of nodes in a single relationship + expect(response.body).to.have.property('edges').length(2); + expect(response.body).to.have.property('messages').length(1); + expect(response.body.messages[0]).equal(ApiMessageCode.ReachedNodesLimit); + }); }); }); } diff --git a/x-pack/test/cloud_security_posture_api/utils.ts b/x-pack/test/cloud_security_posture_api/utils.ts index e64c583af3868..210a081b91473 100644 --- a/x-pack/test/cloud_security_posture_api/utils.ts +++ b/x-pack/test/cloud_security_posture_api/utils.ts @@ -36,22 +36,23 @@ export const waitForPluginInitialized = ({ logger.debug('CSP plugin is initialized'); }); -export function result(status: number): CallbackHandler { +export function result(status: number, logger?: ToolingLog): CallbackHandler { return (err: any, res: Response) => { if ((res?.status || err.status) !== status) { - const e = new Error( + throw new Error( `Expected ${status} ,got ${res?.status || err.status} resp: ${ res?.body ? JSON.stringify(res.body) : err.text }` ); - throw e; + } else if (err) { + logger?.warning(`Error result ${err.text}`); } }; } export class EsIndexDataProvider { private es: EsClient; - private index: string; + private readonly index: string; constructor(es: EsClient, index: string) { this.es = es; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_with_streaming.ts b/x-pack/test/fleet_api_integration/apis/epm/install_with_streaming.ts index 152e3dfd4c69d..4fa0e485be2b5 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_with_streaming.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_with_streaming.ts @@ -36,7 +36,8 @@ export default function (providerContext: FtrProviderContext) { return res?._source?.['epm-packages'] as Installation; }; - describe('Installs a package using stream-based approach', () => { + // Failing: See https://github.com/elastic/kibana/issues/199701 + describe.skip('Installs a package using stream-based approach', () => { skipIfNoDockerRegistry(providerContext); before(async () => { diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts index 51b0842e60bc8..c5927d894911c 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts @@ -114,6 +114,34 @@ export function initRoutes( } ); + router.post( + { + path: `/api/sample_tasks/run_mark_removed_tasks_as_unrecognized`, + validate: { + body: schema.object({}), + }, + }, + async function ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise> { + try { + const taskManager = await taskManagerStart; + await taskManager.ensureScheduled({ + id: 'mark_removed_tasks_as_unrecognized', + taskType: 'task_manager:mark_removed_tasks_as_unrecognized', + schedule: { interval: '1h' }, + state: {}, + params: {}, + }); + return res.ok({ body: await taskManager.runSoon('mark_removed_tasks_as_unrecognized') }); + } catch (err) { + return res.ok({ body: { id: 'mark_removed_tasks_as_unrecognized', error: `${err}` } }); + } + } + ); + router.post( { path: `/api/sample_tasks/bulk_enable`, diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 091e0fe01e415..c8056c2ee205e 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -168,6 +168,7 @@ export default function ({ getService }: FtrProviderContext) { 'security:telemetry-timelines', 'session_cleanup', 'task_manager:delete_inactive_background_task_nodes', + 'task_manager:mark_removed_tasks_as_unrecognized', ]); }); }); diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management_removed_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management_removed_types.ts index aae90a52572c7..a7447353e805a 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management_removed_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management_removed_types.ts @@ -92,6 +92,12 @@ export default function ({ getService }: FtrProviderContext) { let scheduledTaskRuns = 0; let scheduledTaskInstanceRunAt = scheduledTask.runAt; + await request + .post('/api/sample_tasks/run_mark_removed_tasks_as_unrecognized') + .set('kbn-xsrf', 'xxx') + .send({}) + .expect(200); + await retry.try(async () => { const tasks = (await currentTasks()).docs; expect(tasks.length).to.eql(3); diff --git a/x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget/server/init_routes.ts b/x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget/server/init_routes.ts index f1e697399fe09..acdbae0b00337 100644 --- a/x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget/server/init_routes.ts +++ b/x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget/server/init_routes.ts @@ -117,6 +117,34 @@ export function initRoutes( } ); + router.post( + { + path: `/api/sample_tasks/run_mark_removed_tasks_as_unrecognized`, + validate: { + body: schema.object({}), + }, + }, + async function ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise> { + try { + const taskManager = await taskManagerStart; + await taskManager.ensureScheduled({ + id: 'mark_removed_tasks_as_unrecognized', + taskType: 'task_manager:mark_removed_tasks_as_unrecognized', + schedule: { interval: '1h' }, + state: {}, + params: {}, + }); + return res.ok({ body: await taskManager.runSoon('mark_removed_tasks_as_unrecognized') }); + } catch (err) { + return res.ok({ body: { id: 'mark_removed_tasks_as_unrecognized', error: `${err}` } }); + } + } + ); + router.post( { path: `/api/sample_tasks/bulk_enable`, diff --git a/x-pack/test/task_manager_claimer_update_by_query/test_suites/task_manager/task_management_removed_types.ts b/x-pack/test/task_manager_claimer_update_by_query/test_suites/task_manager/task_management_removed_types.ts index aae90a52572c7..a7447353e805a 100644 --- a/x-pack/test/task_manager_claimer_update_by_query/test_suites/task_manager/task_management_removed_types.ts +++ b/x-pack/test/task_manager_claimer_update_by_query/test_suites/task_manager/task_management_removed_types.ts @@ -92,6 +92,12 @@ export default function ({ getService }: FtrProviderContext) { let scheduledTaskRuns = 0; let scheduledTaskInstanceRunAt = scheduledTask.runAt; + await request + .post('/api/sample_tasks/run_mark_removed_tasks_as_unrecognized') + .set('kbn-xsrf', 'xxx') + .send({}) + .expect(200); + await retry.try(async () => { const tasks = (await currentTasks()).docs; expect(tasks.length).to.eql(3); diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts index 741d25291e8fa..aaccdd0e9a41c 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts @@ -12,6 +12,7 @@ import { } from '@kbn/core-http-common'; import { result } from '@kbn/test-suites-xpack/cloud_security_posture_api/utils'; import type { Agent } from 'supertest'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/v1'; import type { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { @@ -19,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const roleScopedSupertest = getService('roleScopedSupertest'); let supertestViewer: Pick; - const postGraph = (agent: Pick, body: any) => { + const postGraph = (agent: Pick, body: GraphRequest) => { const req = agent .post('/internal/cloud_security_posture/graph') .set(ELASTIC_HTTP_VERSION_HEADER, '1') @@ -48,7 +49,6 @@ export default function ({ getService }: FtrProviderContext) { it('should return an empty graph', async () => { const response = await postGraph(supertestViewer, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -57,20 +57,26 @@ export default function ({ getService }: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(0); expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); }); it('should return a graph with nodes and edges by actor', async () => { const response = await postGraph(supertestViewer, { query: { - actorIds: ['admin@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [{ match_phrase: { 'actor.entity.id': 'admin@example.com' } }], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); diff --git a/yarn.lock b/yarn.lock index 5faf426cf4d25..47ee0df930999 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1721,12 +1721,12 @@ "@elastic/transport" "^8.3.1" tslib "^2.4.0" -"@elastic/elasticsearch@^8.15.0": - version "8.15.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.15.0.tgz#cb29b3ae33203c545d435cf3dc4b557c8b4961d5" - integrity sha512-mG90EMdTDoT6GFSdqpUAhWK9LGuiJo6tOWqs0Usd/t15mPQDj7ZqHXfCBqNkASZpwPZpbAYVjd57S6nbUBINCg== +"@elastic/elasticsearch@^8.15.1": + version "8.15.1" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.15.1.tgz#ca294ba11ed1514bf87d4a2e253b11f6cefd8552" + integrity sha512-L3YzSaxrasMMGtcxnktiUDjS5f177L0zpHsBH+jL0LgPhdMk9xN/VKrAaYzvri86IlV5IbveA0ANV6o/BDUmhQ== dependencies: - "@elastic/transport" "^8.7.0" + "@elastic/transport" "^8.8.1" tslib "^2.4.0" "@elastic/ems-client@8.5.3": @@ -1906,10 +1906,10 @@ undici "^5.28.3" yaml "^2.2.2" -"@elastic/transport@^8.3.1", "@elastic/transport@^8.7.0": - version "8.7.0" - resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.7.0.tgz#006987fc5583f61c266e0b1003371e82efc7a6b5" - integrity sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A== +"@elastic/transport@^8.3.1", "@elastic/transport@^8.8.1": + version "8.8.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.8.1.tgz#d64244907bccdad5626c860b492faeef12194b1f" + integrity sha512-4RQIiChwNIx3B0O+2JdmTq/Qobj6+1g2RQnSv1gt4V2SVfAYjGwOKu0ZMKEHQOXYNG6+j/Chero2G9k3/wXLEw== dependencies: "@opentelemetry/api" "1.x" debug "^4.3.4" @@ -8483,12 +8483,12 @@ require-from-string "^2.0.2" uri-js-replace "^1.0.1" -"@redocly/cli@^1.25.10": - version "1.25.10" - resolved "https://registry.yarnpkg.com/@redocly/cli/-/cli-1.25.10.tgz#647e33e4171d74a4f879304ba87366ac650ed83d" - integrity sha512-zoRMvSYOLzurcb3be5HLLlc5dLGICyHY8mueCbdE2DmLbFERhJJ5iiABKvNRJSr03AR6X569f4mraBJpAsGJnQ== +"@redocly/cli@^1.25.11": + version "1.25.11" + resolved "https://registry.yarnpkg.com/@redocly/cli/-/cli-1.25.11.tgz#8ec17a6535aebfd166e8cab8ffcc9d768af1b014" + integrity sha512-dttBsmLnnbTlJCTa+s7Sy+qtXDq692n7Ru3nUUIHp9XdCbhXIHWhpc8uAl+GmR4MGbVe8ohATl3J+zX3aFy82A== dependencies: - "@redocly/openapi-core" "1.25.10" + "@redocly/openapi-core" "1.25.11" abort-controller "^3.0.0" chokidar "^3.5.1" colorette "^1.2.0" @@ -8513,10 +8513,10 @@ resolved "https://registry.yarnpkg.com/@redocly/config/-/config-0.16.0.tgz#4b7700a5cb6e04bc6d6fdb94b871c9e260a1fba6" integrity sha512-t9jnODbUcuANRSl/K4L9nb12V+U5acIHnVSl26NWrtSdDZVtoqUXk2yGFPZzohYf62cCfEQUT8ouJ3bhPfpnJg== -"@redocly/openapi-core@1.25.10", "@redocly/openapi-core@^1.4.0": - version "1.25.10" - resolved "https://registry.yarnpkg.com/@redocly/openapi-core/-/openapi-core-1.25.10.tgz#6ca3f1ad1b826e3680f91752abf11aa40856f6b8" - integrity sha512-wcGnSonJZvjpPaJJs+qh0ADYy0aCbaNhCXhJVES9RlknMc7V9nbqLQ67lkwaXhpp/fskm9GJWL/U9Xyiuclbqw== +"@redocly/openapi-core@1.25.11", "@redocly/openapi-core@^1.4.0": + version "1.25.11" + resolved "https://registry.yarnpkg.com/@redocly/openapi-core/-/openapi-core-1.25.11.tgz#93f168284986da6809363b001e9aa7c2104c2fc0" + integrity sha512-bH+a8izQz4fnKROKoX3bEU8sQ9rjvEIZOqU6qTmxlhOJ0NsKa5e+LmU18SV0oFeg5YhWQhhEDihXkvKJ1wMMNQ== dependencies: "@redocly/ajv" "^8.11.2" "@redocly/config" "^0.16.0"