From a2f4c8f4bb78681566f16fdce914c4a989d12bbc Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Wed, 24 Jul 2024 10:10:47 -0400 Subject: [PATCH 01/70] interim commit - undo me --- .../server/utils/create_persistence_rule_type_wrapper.ts | 8 +++++--- .../server/lib/detection_engine/rule_types/eql/eql.ts | 7 +++++++ .../query/alert_suppression/group_and_bulk_create.ts | 2 ++ .../server/lib/detection_engine/rule_types/query/query.ts | 1 + .../utils/bulk_create_suppressed_alerts_in_memory.ts | 3 ++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 58ec5ea0818d1..bc7ca1949231c 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -471,9 +471,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, {}); // filter out alerts that were already suppressed - // alert was suppressed if its suppression ends is older than suppression end of existing alert - // if existing alert was created earlier during the same rule execution - then alerts can be counted as not suppressed yet - // as they are processed for the first against this existing alert + // alert was suppressed if its suppression ends is older + // than suppression end of existing alert + // if existing alert was created earlier during the same + // rule execution - then alerts can be counted as not suppressed yet + // as they are processed for the first time against this existing alert const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { const existingAlert = existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 61c4f1b95dfd8..b672f36f61539 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -129,6 +129,13 @@ export const eqlExecutor = async ({ const { events, sequences } = response.hits; + console.error('SEQUENCES LENGTH', sequences?.length); + + sequences?.forEach((sequence) => { + console.error('sequence length', sequence.events?.length); + console.error('sequence join keys length', sequence.join_keys?.length); + }); + if (events) { if (isAlertSuppressionActive) { await bulkCreateSuppressedAlertsInMemory({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index b1d897250ca4e..c88ce85c9b864 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -250,6 +250,8 @@ export const groupAndBulkCreate = async ({ terms: Object.entries(bucket.key).map(([key, value]) => ({ field: key, value })), })); + console.error('SUPPRESSION BUCKETS', JSON.stringify(suppressionBuckets, null, 2)); + const wrappedAlerts = wrapSuppressedAlerts({ suppressionBuckets, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts index 272184dbf1e58..6c08eb5fca96c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts @@ -66,6 +66,7 @@ export const queryExecutor = async ({ const hasPlatinumLicense = license.hasAtLeast('platinum'); const result = + // TODO: replace this with getIsAlertSuppressionActive function ruleParams.alertSuppression?.groupBy != null && hasPlatinumLicense ? await groupAndBulkCreate({ runOpts, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 030cb213d94dd..a1e1a7f4c6836 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -56,7 +56,8 @@ export interface BulkCreateSuppressedAlertsParams } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. - * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, regular alerts will be created for such events without suppression + * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, + * regular alerts will be created for such events without suppression */ export const bulkCreateSuppressedAlertsInMemory = async ({ enrichedEvents, From 1eb6a580803e4a180579f9a5933505ee0e2022a9 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 1 Aug 2024 10:47:42 -0400 Subject: [PATCH 02/70] first commit, working suppression by time range, need to filter out building block alerts on first alert creation --- .../create_persistence_rule_type_wrapper.ts | 513 ++++++++++-------- .../server/utils/persistence_types.ts | 3 +- .../eql/build_alert_group_from_sequence.ts | 6 +- .../rule_types/eql/create_eql_alert_type.ts | 4 +- .../detection_engine/rule_types/eql/eql.ts | 42 +- .../lib/detection_engine/rule_types/types.ts | 3 +- ...bulk_create_suppressed_alerts_in_memory.ts | 22 +- .../utils/bulk_create_with_suppression.ts | 12 +- .../utils/get_is_alert_suppression_active.ts | 5 + .../rule_types/utils/suppression_utils.ts | 18 +- .../utils/wrap_suppressed_alerts.ts | 54 +- .../lib/detection_engine/scripts/post_rule.sh | 1 + .../rules/queries/sequence_eql_query.json | 68 +++ 13 files changed, 486 insertions(+), 265 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index bc7ca1949231c..db25683e48a4f 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -61,6 +61,10 @@ const augmentAlerts = ({ kibanaVersion: string; currentTimeOverride: Date | undefined; }) => { + console.error( + 'DO AUGMENTED ALERTS HAVE INSTANCE IDS', + alerts.map((alert) => alert._source[ALERT_INSTANCE_ID]) + ); const commonRuleFields = getCommonAlertFields(options); return alerts.map((alert) => { return { @@ -81,7 +85,14 @@ const augmentAlerts = ({ }; const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => { - return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); + const sourceNoId = (alertSource) => { + if (Object.hasOwn(alertSource, '_id')) { + const { _id, _index, _source, ...source } = alertSource; + return { ..._source, ...source }; + } + return alertSource; + }; + return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, sourceNoId(alert._source)]); //sourceNoId(alert._source)]); }; /** @@ -206,6 +217,8 @@ export const isExistingDateGtEqThanAlert = < property: typeof ALERT_SUPPRESSION_END | typeof ALERT_SUPPRESSION_START ) => { const existingDate = existingAlert?._source?.[property]; + console.error('WHAT IS THE EXISTING DATE', existingDate); + console.error('WHAT IS THE SOURCE PROPERTY', alert._source[property]); return existingDate ? existingDate >= alert._source[property].toISOString() : false; }; @@ -370,212 +383,304 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper enrichAlerts, currentTimeOverride, isRuleExecutionOnly, - maxAlerts + maxAlerts, + buildingBlockAlerts ) => { - const ruleDataClientWriter = await ruleDataClient.getWriter({ - namespace: options.spaceId, - }); + try { + console.error( + 'ALERT WITH SUPPRESSION INSTANCE IDS', + alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) + ); + const ruleDataClientWriter = await ruleDataClient.getWriter({ + namespace: options.spaceId, + }); - // Only write alerts if: - // - writing is enabled - // AND - // - rule execution has not been cancelled due to timeout - // OR - // - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway - const writeAlerts = - ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); + // Only write alerts if: + // - writing is enabled + // AND + // - rule execution has not been cancelled due to timeout + // OR + // - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway + const writeAlerts = + ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); - let alertsWereTruncated = false; + let alertsWereTruncated = false; - if (writeAlerts && alerts.length > 0) { - const suppressionWindowStart = dateMath.parse(suppressionWindow, { - forceNow: currentTimeOverride, - }); + if (writeAlerts && alerts.length > 0) { + const suppressionWindowStart = dateMath.parse(suppressionWindow, { + forceNow: currentTimeOverride, + }); - if (!suppressionWindowStart) { - throw new Error('Failed to parse suppression window'); - } + if (!suppressionWindowStart) { + throw new Error('Failed to parse suppression window'); + } - const filteredDuplicates = await filterDuplicateAlerts({ - alerts, - ruleDataClient, - spaceId: options.spaceId, - }); + const filteredDuplicates = await filterDuplicateAlerts({ + alerts, + ruleDataClient, + spaceId: options.spaceId, + }); - if (filteredDuplicates.length === 0) { - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated, - }; - } + if (filteredDuplicates.length === 0) { + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated, + }; + } - const suppressionAlertSearchRequest = { - body: { - size: filteredDuplicates.length, - query: { - bool: { - filter: [ - { - range: { - [ALERT_START]: { - gte: suppressionWindowStart.toISOString(), + const suppressionAlertSearchRequest = { + body: { + size: filteredDuplicates.length, + query: { + bool: { + filter: [ + { + range: { + [ALERT_START]: { + gte: suppressionWindowStart.toISOString(), + }, }, }, - }, - { - terms: { - [ALERT_INSTANCE_ID]: filteredDuplicates.map( - (alert) => alert._source[ALERT_INSTANCE_ID] - ), + { + terms: { + [ALERT_INSTANCE_ID]: filteredDuplicates.map( + (alert) => alert._source[ALERT_INSTANCE_ID] + ), + }, }, - }, - { - bool: { - must_not: { - term: { - [ALERT_WORKFLOW_STATUS]: 'closed', + { + bool: { + must_not: { + term: { + [ALERT_WORKFLOW_STATUS]: 'closed', + }, }, }, }, - }, - ], - }, - }, - collapse: { - field: ALERT_INSTANCE_ID, - }, - sort: [ - { - [ALERT_START]: { - order: 'desc' as const, + ], }, }, - ], - }, - }; - - const response = await ruleDataClient - .getReader({ namespace: options.spaceId }) - .search< - typeof suppressionAlertSearchRequest, - BackendAlertWithSuppressionFields870<{}> - >(suppressionAlertSearchRequest); - - const existingAlertsByInstanceId = response.hits.hits.reduce< - Record>> - >((acc, hit) => { - acc[hit._source[ALERT_INSTANCE_ID]] = hit; - return acc; - }, {}); - - // filter out alerts that were already suppressed - // alert was suppressed if its suppression ends is older - // than suppression end of existing alert - // if existing alert was created earlier during the same - // rule execution - then alerts can be counted as not suppressed yet - // as they are processed for the first time against this existing alert - const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - - if ( - !existingAlert || - existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId - ) { - return true; - } - - return !isExistingDateGtEqThanAlert(existingAlert, alert, ALERT_SUPPRESSION_END); - }); - - if (nonSuppressedAlerts.length === 0) { - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated, + collapse: { + field: ALERT_INSTANCE_ID, + }, + sort: [ + { + [ALERT_START]: { + order: 'desc' as const, + }, + }, + ], + }, }; - } - - const { alertCandidates, suppressedAlerts: suppressedInMemoryAlerts } = - suppressAlertsInMemory(nonSuppressedAlerts); - const [duplicateAlerts, newAlerts] = partition(alertCandidates, (alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - - // if suppression enabled only on rule execution, we need to suppress alerts only against - // alert created in the same rule execution. Otherwise, we need to create a new alert to accommodate per rule execution suppression - if (isRuleExecutionOnly) { - return ( + const response = await ruleDataClient + .getReader({ namespace: options.spaceId }) + .search< + typeof suppressionAlertSearchRequest, + BackendAlertWithSuppressionFields870<{}> + >(suppressionAlertSearchRequest); + + const existingAlertsByInstanceId = response.hits.hits.reduce< + Record>> + >((acc, hit) => { + acc[hit._source[ALERT_INSTANCE_ID]] = hit; + return acc; + }, {}); + + // filter out alerts that were already suppressed + // alert was suppressed if its suppression ends is older + // than suppression end of existing alert + // if existing alert was created earlier during the same + // rule execution - then alerts can be counted as not suppressed yet + // as they are processed for the first time against this existing alert + const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + if ( + !existingAlert || existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId + ) { + return true; + } + + return !isExistingDateGtEqThanAlert( + existingAlert, + alert, + ALERT_SUPPRESSION_END ); - } else { - return existingAlert != null; + }); + + if (nonSuppressedAlerts.length === 0) { + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated, + }; } - }); - const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - const existingDocsCount = - existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; - - return [ - { - update: { - _id: existingAlert._id, - _index: existingAlert._index, - require_alias: false, + const { alertCandidates, suppressedAlerts: suppressedInMemoryAlerts } = + suppressAlertsInMemory(nonSuppressedAlerts); + + const [duplicateAlerts, newAlerts] = partition(alertCandidates, (alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + // if suppression enabled only on rule execution, we need to suppress alerts only against + // alert created in the same rule execution. Otherwise, we need to create a new alert to accommodate per rule execution suppression + if (isRuleExecutionOnly) { + return ( + existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId + ); + } else { + return existingAlert != null; + } + }); + + console.error('ARE THERE DUPLICATE ALERTS', duplicateAlerts?.length); + + const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + const existingDocsCount = + existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; + + return [ + { + update: { + _id: existingAlert._id, + _index: existingAlert._index, + require_alias: false, + }, }, - }, - { - doc: { - ...getUpdatedSuppressionBoundaries( - existingAlert, - alert, - options.executionId - ), - [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), - [ALERT_SUPPRESSION_DOCS_COUNT]: - existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, + { + doc: { + ...getUpdatedSuppressionBoundaries( + existingAlert, + alert, + options.executionId + ), + [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), + [ALERT_SUPPRESSION_DOCS_COUNT]: + existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, + }, }, - }, - ]; - }); + ]; + }); + + let enrichedAlerts = newAlerts; + console.error('WHAT ARE ENRICHED ALERTS LENGTH BEFORE', enrichedAlerts?.length); + + if (enrichAlerts) { + try { + enrichedAlerts = await enrichAlerts(enrichedAlerts, { + spaceId: options.spaceId, + }); + } catch (e) { + logger.debug('Enrichments failed'); + } + } - let enrichedAlerts = newAlerts; + console.error('WHAT ARE ENRICHED ALERTS LENGTH AFTER', enrichedAlerts?.length); - if (enrichAlerts) { - try { - enrichedAlerts = await enrichAlerts(enrichedAlerts, { - spaceId: options.spaceId, - }); - } catch (e) { - logger.debug('Enrichments failed'); + if (maxAlerts && enrichedAlerts.length > maxAlerts) { + enrichedAlerts.length = maxAlerts; + alertsWereTruncated = true; } - } - if (maxAlerts && enrichedAlerts.length > maxAlerts) { - enrichedAlerts.length = maxAlerts; - alertsWereTruncated = true; - } + const augmentedAlerts = augmentAlerts({ + alerts: enrichedAlerts, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + currentTimeOverride, + }); + + const augmentedBuildingBlockAlerts = + newAlerts?.length > 0 && buildingBlockAlerts?.length > 0 + ? augmentAlerts({ + alerts: buildingBlockAlerts, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + currentTimeOverride, + }) + : []; + + console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); + console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); + console.error( + 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', + JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) + ); + const bulkResponse = await ruleDataClientWriter.bulk({ + body: [ + ...duplicateAlertUpdates, + ...mapAlertsToBulkCreate(augmentedAlerts), + ...(newAlerts?.length > 0 + ? mapAlertsToBulkCreate(augmentedBuildingBlockAlerts) + : []), + ], + refresh: true, + }); - const augmentedAlerts = augmentAlerts({ - alerts: enrichedAlerts, - options, - kibanaVersion: ruleDataClient.kibanaVersion, - currentTimeOverride, - }); + if (bulkResponse == null) { + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated: false, + }; + } - const bulkResponse = await ruleDataClientWriter.bulk({ - body: [...duplicateAlertUpdates, ...mapAlertsToBulkCreate(augmentedAlerts)], - refresh: true, - }); + const createdAlerts = augmentedAlerts + .map((alert, idx) => { + const responseItem = + bulkResponse.body.items[idx + duplicateAlerts.length].create; + return { + _id: responseItem?._id ?? '', + _index: responseItem?._index ?? '', + ...alert._source, + }; + }) + .filter( + (_, idx) => + bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 + ) + // Security solution's EQL rule consists of building block alerts which should be filtered out. + // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. + .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); + + createdAlerts.forEach((alert) => + options.services.alertFactory + .create(alert._id) + .replaceState({ + signals_count: 1, + }) + .scheduleActions(type.defaultActionGroupId, { + rule: mapKeys(snakeCase, { + ...options.params, + name: options.rule.name, + id: options.rule.id, + }), + results_link: type.getViewInAppRelativeUrl?.({ + rule: { ...options.rule, params: options.params }, + start: Date.parse(alert[TIMESTAMP]), + end: Date.parse(alert[TIMESTAMP]), + }), + alerts: [formatAlert?.(alert) ?? alert], + }) + ); - if (bulkResponse == null) { + return { + createdAlerts, + suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts], + errors: errorAggregator(bulkResponse.body, [409]), + alertsWereTruncated, + }; + } else { + logger.debug('Writing is disabled.'); return { createdAlerts: [], errors: {}, @@ -583,60 +688,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alertsWereTruncated: false, }; } - - const createdAlerts = augmentedAlerts - .map((alert, idx) => { - const responseItem = - bulkResponse.body.items[idx + duplicateAlerts.length].create; - return { - _id: responseItem?._id ?? '', - _index: responseItem?._index ?? '', - ...alert._source, - }; - }) - .filter( - (_, idx) => - bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 - ) - // Security solution's EQL rule consists of building block alerts which should be filtered out. - // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. - .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); - - createdAlerts.forEach((alert) => - options.services.alertFactory - .create(alert._id) - .replaceState({ - signals_count: 1, - }) - .scheduleActions(type.defaultActionGroupId, { - rule: mapKeys(snakeCase, { - ...options.params, - name: options.rule.name, - id: options.rule.id, - }), - results_link: type.getViewInAppRelativeUrl?.({ - rule: { ...options.rule, params: options.params }, - start: Date.parse(alert[TIMESTAMP]), - end: Date.parse(alert[TIMESTAMP]), - }), - alerts: [formatAlert?.(alert) ?? alert], - }) - ); - - return { - createdAlerts, - suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts], - errors: errorAggregator(bulkResponse.body, [409]), - alertsWereTruncated, - }; - } else { - logger.debug('Writing is disabled.'); - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated: false, - }; + } catch (exc) { + console.error('BIG TIME EXCEPTION', exc); } }, }, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 1ff6a6e62d743..f8a6b26ed89f8 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -54,7 +54,8 @@ export type SuppressedAlertService = ( ) => Promise>, currentTimeOverride?: Date, isRuleExecutionOnly?: boolean, - maxAlerts?: number + maxAlerts?: number, + buildingBlockAlerts?: Array<{ _id: string; _source: T }> ) => Promise>; export interface SuppressedAlertServiceResult extends PersistenceAlertServiceResult { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 2675c3996e865..4e91af23dc91b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -54,8 +54,10 @@ export const buildAlertGroupFromSequence = ( return []; } - // The "building block" alerts start out as regular BaseFields. We'll add the group ID and index fields - // after creating the shell alert later on, since that's when the group ID is determined. + // The "building block" alerts start out as regular BaseFields. + // We'll add the group ID and index fields + // after creating the shell alert later on + // since that's when the group ID is determined. let baseAlerts: BaseFieldsLatest[] = []; try { baseAlerts = sequence.events.map((event) => diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 81971feeecfc1..a8bae32e77260 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -86,7 +86,8 @@ export const createEqlAlertType = ( const wrapSuppressedHits = ( events: SignalSourceHit[], - buildReasonMessage: BuildReasonMessage + buildReasonMessage: BuildReasonMessage, + skipGenerateId: boolean ) => wrapSuppressedAlerts({ events, @@ -98,6 +99,7 @@ export const createEqlAlertType = ( alertTimestampOverride, ruleExecutionLogger, publicBaseUrl, + skipGenerateId, primaryTimestamp, secondaryTimestamp, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 73e278e76ae51..35a1b75aff0fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -5,6 +5,8 @@ * 2.0. */ import { performance } from 'perf_hooks'; +import { partition } from 'lodash'; + import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { @@ -45,6 +47,11 @@ import type { import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; +import { + ALERT_ANCESTORS, + ALERT_BUILDING_BLOCK_TYPE, + ALERT_GROUP_ID, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; interface EqlExecutorParams { inputIndex: string[]; @@ -162,12 +169,45 @@ export const eqlExecutor = async ({ newSignals = wrapHits(events, buildReasonMessageForEqlAlert); } } else if (sequences) { + console.error('isAlertSuppressionActive??? ', isAlertSuppressionActive); if (isAlertSuppressionActive) { result.warningMessages.push( 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' ); + + const candidateSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); + // partition sequence alert from building block alerts + const [sequenceAlerts, buildingBlockAlerts] = partition( + candidateSignals, + (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null + ); + console.error( + 'WHAT ARE THE NEW SIGNALS', + sequenceAlerts.map((alert) => alert._source['agent.name']) + ); + await bulkCreateSuppressedAlertsInMemory({ + enrichedEvents: sequenceAlerts, + buildingBlockAlerts, + skipWrapping: true, + toReturn: result, + wrapHits, + bulkCreate, + services, + buildReasonMessage: buildReasonMessageForEqlAlert, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + }); + } else { + newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } - newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); + // once partitioned, we pass in the sequence alerts to check for suppression + // and then filter out the suppressable sequence alerts and the building + // block alerts associated with the suppressable sequence alerts. } else { throw new Error( 'eql query response should have either `sequences` or `events` but had neither' diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 8f7a50b195e4f..1c795e1ba94f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -341,7 +341,8 @@ export type WrapHits = ( export type WrapSuppressedHits = ( hits: Array>, - buildReasonMessage: BuildReasonMessage + buildReasonMessage: BuildReasonMessage, + skipGenerateId: boolean ) => Array>; export type WrapSequences = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index a1e1a7f4c6836..86eb38613dbae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -7,6 +7,7 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType, @@ -49,10 +50,12 @@ export interface BulkCreateSuppressedAlertsParams | 'alertTimestampOverride' > { enrichedEvents: SignalSourceHit[]; + buildingBlockAlerts?: Array>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; + skipWrapping?: boolean; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -61,6 +64,8 @@ export interface BulkCreateSuppressedAlertsParams */ export const bulkCreateSuppressedAlertsInMemory = async ({ enrichedEvents, + buildingBlockAlerts, + skipWrapping = false, toReturn, wrapHits, bulkCreate, @@ -92,14 +97,26 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage); + console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); suppressibleEvents = partitionedEvents[0]; } - const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage); + // refactor the below into a separate function + const suppressibleWrappedDocs = wrapSuppressedHits( + suppressibleEvents, + buildReasonMessage, + skipWrapping + ); + + console.error( + 'SUPPRESSIBLE WRAPPED instance ids', + suppressibleWrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) + ); return executeBulkCreateAlerts({ suppressibleWrappedDocs, unsuppressibleWrappedDocs, + buildingBlockAlerts, toReturn, bulkCreate, services, @@ -126,6 +143,7 @@ export interface ExecuteBulkCreateAlertsParams { unsuppressibleWrappedDocs: Array>; suppressibleWrappedDocs: Array>; + buildingBlockAlerts?: Array>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; maxNumberOfAlertsMultiplier?: number; @@ -139,6 +157,7 @@ export const executeBulkCreateAlerts = async < >({ unsuppressibleWrappedDocs, suppressibleWrappedDocs, + buildingBlockAlerts, toReturn, bulkCreate, services, @@ -177,6 +196,7 @@ export const executeBulkCreateAlerts = async < alertWithSuppression, ruleExecutionLogger, wrappedDocs: suppressibleWrappedDocs, + buildingBlockAlerts, services, suppressionWindow, alertTimestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 75aa46d039277..8d676f00e0e31 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -22,6 +22,7 @@ import type { import type { RuleServices } from '../types'; import { createEnrichEventsFunction } from './enrichments'; import type { ExperimentalFeatures } from '../../../../../common'; +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; export interface GenericBulkCreateResponse { success: boolean; @@ -40,6 +41,7 @@ export const bulkCreateWithSuppression = async < alertWithSuppression, ruleExecutionLogger, wrappedDocs, + buildingBlockAlerts, services, suppressionWindow, alertTimestampOverride, @@ -50,6 +52,7 @@ export const bulkCreateWithSuppression = async < alertWithSuppression: SuppressedAlertService; ruleExecutionLogger: IRuleExecutionLogForExecutors; wrappedDocs: Array>; + buildingBlockAlerts?: Array>; services: RuleServices; suppressionWindow: string; alertTimestampOverride: Date | undefined; @@ -92,6 +95,11 @@ export const bulkCreateWithSuppression = async < } }; + console.error( + 'DO WRAPPED DOCS HAVE INSTANCE IDS', + wrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) + ); + const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( wrappedDocs.map((doc) => ({ @@ -99,11 +107,13 @@ export const bulkCreateWithSuppression = async < // `fields` should have already been merged into `doc._source` _source: doc._source, })), + // add building block alerts here when you get back suppressionWindow, enrichAlertsWrapper, alertTimestampOverride, isSuppressionPerRuleExecution, - maxAlerts + maxAlerts, + buildingBlockAlerts // do the same map as wrappedDocs ); const end = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts index 5ab06db3043af..64783f86e5d91 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts @@ -28,12 +28,15 @@ export const getIsAlertSuppressionActive = async ({ alertSuppression, isFeatureDisabled = false, }: GetIsAlertSuppressionActiveParams) => { + console.error('IS IT DISABLED', isFeatureDisabled); if (isFeatureDisabled) { return false; } const isAlertSuppressionConfigured = Boolean(alertSuppression?.groupBy?.length); + console.error('IS IT CONFIGURED', isAlertSuppressionConfigured); + if (!isAlertSuppressionConfigured) { return false; } @@ -41,5 +44,7 @@ export const getIsAlertSuppressionActive = async ({ const license = await firstValueFrom(licensing.license$); const hasPlatinumLicense = license.hasAtLeast('platinum'); + console.error('hasPlatinumLicense?? ', hasPlatinumLicense); + return hasPlatinumLicense; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 44febba73e68e..d9b6091629021 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -17,6 +17,7 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; +import { SignalSourceHit } from '../types'; export interface SuppressionTerm { field: string; @@ -33,6 +34,7 @@ export const getSuppressionAlertFields = ({ suppressionTerms, fallbackTimestamp, instanceId, + event, }: { fields: Record | undefined; primaryTimestamp: string; @@ -40,10 +42,16 @@ export const getSuppressionAlertFields = ({ suppressionTerms: SuppressionTerm[]; fallbackTimestamp: string; instanceId: string; + event?: SignalSourceHit; }) => { + console.error( + `primary timestamp: ${primaryTimestamp}, secondaryTimestamp: ${secondaryTimestamp}, fallbackTimestamp: ${fallbackTimestamp}` + ); + console.error('WHAT ARE FIELDS', JSON.stringify(fields)); + console.error('WHAT ARE EVENT', JSON.stringify(event)); const suppressionTime = new Date( - get(fields, primaryTimestamp) ?? - (secondaryTimestamp && get(fields, secondaryTimestamp)) ?? + get(fields ?? event?._source, primaryTimestamp) ?? + (secondaryTimestamp && get(fields ?? event?._source, secondaryTimestamp)) ?? fallbackTimestamp ); @@ -55,6 +63,8 @@ export const getSuppressionAlertFields = ({ [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }; + console.error('WHAT ARE THE SUPPRESSION FIELDS', JSON.stringify(suppressionFields)); + return suppressionFields; }; @@ -64,13 +74,15 @@ export const getSuppressionAlertFields = ({ export const getSuppressionTerms = ({ alertSuppression, fields, + event, }: { fields: Record | undefined; alertSuppression: AlertSuppressionCamel | undefined; + event: SignalSourceHit | undefined; }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - const suppressedProps = pick(fields, suppressedBy) as Record< + const suppressedProps = pick(fields ?? event?._source, suppressedBy) as Record< string, string[] | number[] | undefined >; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 70fee20116fc4..07af55efa0667 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -47,6 +47,7 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger, publicBaseUrl, primaryTimestamp, + skipGenerateId = false, secondaryTimestamp, }: { events: SignalSourceHit[]; @@ -59,39 +60,43 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger: IRuleExecutionLogForExecutors; publicBaseUrl: string | undefined; primaryTimestamp: string; + skipGenerateId: boolean; secondaryTimestamp?: string; }): Array> => { return events.map((event) => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, fields: event.fields, + event, }); - - const id = generateId( - event._index, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - event._id!, - String(event._version), - `${spaceId}:${completeRule.alertId}` - ); + let id = event._id; + let baseAlert: BaseFieldsLatest = event; + if (!skipGenerateId) { + id = generateId( + event._index, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + event._id!, + String(event._version), + `${spaceId}:${completeRule.alertId}` + ); + baseAlert = buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + } const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - - const baseAlert: BaseFieldsLatest = buildBulkBody( - spaceId, - completeRule, - event, - mergeStrategy, - [], - true, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - ruleExecutionLogger, - id, - publicBaseUrl - ); - + console.error('INSTANCE ID', instanceId); return { _id: id, _index: '', @@ -101,6 +106,7 @@ export const wrapSuppressedAlerts = ({ primaryTimestamp, secondaryTimestamp, fields: event.fields, + event, suppressionTerms, fallbackTimestamp: baseAlert[TIMESTAMP], instanceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh index e8f5c627d3c88..b40c416943295 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh @@ -22,6 +22,7 @@ do { curl -s -k \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: 123' \ + -H 'elastic-api-version: 2023-10-31' \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/rules \ -d @${RULE} \ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json new file mode 100644 index 0000000000000..fa49022183d78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -0,0 +1,68 @@ +{ + "name": "EQL sequence rule", + "description": "Rule with an eql query", + "false_positives": [ + "https://www.example.com/some-article-about-a-false-positive", + "some text string about why another condition could be a false positive" + ], + "rule_id": "rule-id-eql-2", + "enabled": true, + "index": ["auditbeat*", "packetbeat*"], + "interval": "30s", + "query": "sequence with maxspan=10m [any where agent.type == \"auditbeat\"] [any where event.category == \"network_traffic\"]", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 99, + "to": "now", + "from": "now-90s", + "severity": "high", + "type": "eql", + "language": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": [ + "http://www.example.com/some-article-about-attack", + "Some plain text string here explaining why this is a valid thing to look out for" + ], + "alert_suppression": { + "group_by": ["agent.name"], + "duration": { "value": 5, "unit": "h" }, + "missing_fields_strategy": "suppress" + }, + "version": 1 +} From 1514ad54c1b03b2208fa9f9440481c0ff73bd350 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 5 Aug 2024 12:13:10 -0400 Subject: [PATCH 03/70] create building block alerts for non-suppressed alert --- .../create_persistence_rule_type_wrapper.ts | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index db25683e48a4f..65b64252b78cb 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -27,6 +27,7 @@ import { ALERT_RULE_EXECUTION_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; +import { ALERT_GROUP_ID } from '@kbn/security-solution-plugin/common/field_maps/field_names'; import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; @@ -598,22 +599,39 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); + console.error( + 'WHAT IS AUGMENTED ALERT', + augmentedAlerts.map( + (augAlert) => augAlert._source._source['kibana.alert.group.id'] + ) + ); + + const matchingBuildingBlockAlerts = buildingBlockAlerts?.filter((someAlert) => { + // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); + // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); + + return ( + someAlert?._source[ALERT_GROUP_ID] === + newAlerts[0]?._source._source['kibana.alert.group.id'] + ); + }); + const augmentedBuildingBlockAlerts = newAlerts?.length > 0 && buildingBlockAlerts?.length > 0 ? augmentAlerts({ - alerts: buildingBlockAlerts, + alerts: matchingBuildingBlockAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, }) : []; - console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); - console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); - console.error( - 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', - JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) - ); + // console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); + // console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); + // console.error( + // 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', + // JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) + // ); const bulkResponse = await ruleDataClientWriter.bulk({ body: [ ...duplicateAlertUpdates, From bb120aaa44f8c79b734b57d29be045c0b5a765a4 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 5 Aug 2024 16:48:10 -0400 Subject: [PATCH 04/70] enables UI for creating suppressed sequence alert --- .../components/step_define_rule/index.tsx | 20 +++---------------- .../components/step_define_rule/schema.tsx | 9 --------- .../step_define_rule/translations.tsx | 16 --------------- .../detection_engine/rule_types/eql/eql.ts | 6 +++--- 4 files changed, 6 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 1ef86c1cab1ed..28b0ad039b173 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -77,7 +77,6 @@ import { isThresholdRule as getIsThresholdRule, isQueryRule, isEsqlRule, - isEqlSequenceQuery, isSuppressionRuleInGA, } from '../../../../../common/detection_engine/utils'; import { EqlQueryBar } from '../eql_query_bar'; @@ -474,11 +473,6 @@ const StepDefineRuleComponent: FC = ({ * purpose and so are treated as if the field is always selected. */ const areSuppressionFieldsSelected = isThresholdRule || groupByFields.length > 0; - const areSuppressionFieldsDisabledBySequence = - isEqlRule(ruleType) && - isEqlSequenceQuery(queryBar?.query?.query as string) && - groupByFields.length === 0; - /** If we don't have ML field information, users can't meaningfully interact with suppression fields */ const areSuppressionFieldsDisabledByMlFields = isMlRule(ruleType) && (mlRuleConfigLoading || !mlSuppressionFields.length); @@ -491,26 +485,18 @@ const StepDefineRuleComponent: FC = ({ * - ML Field information is not available */ const areSuppressionFieldsDisabled = - !isAlertSuppressionLicenseValid || - areSuppressionFieldsDisabledBySequence || - areSuppressionFieldsDisabledByMlFields; + !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledByMlFields; const isSuppressionGroupByDisabled = (areSuppressionFieldsDisabled || isEsqlSuppressionLoading) && !areSuppressionFieldsSelected; const suppressionGroupByDisabledText = useMemo(() => { - if (areSuppressionFieldsDisabledBySequence) { - return i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP; - } else if (areSuppressionFieldsDisabledByMlFields) { + if (areSuppressionFieldsDisabledByMlFields) { return i18n.MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL; } else { return alertSuppressionUpsellingMessage; } - }, [ - alertSuppressionUpsellingMessage, - areSuppressionFieldsDisabledByMlFields, - areSuppressionFieldsDisabledBySequence, - ]); + }, [alertSuppressionUpsellingMessage, areSuppressionFieldsDisabledByMlFields]); const suppressionGroupByFields = useMemo(() => { if (isEsqlRule(ruleType)) { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index bdf14f4b6fd4a..b80a569bf1754 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -18,7 +18,6 @@ import { } from '../../../../common/components/threat_match/helpers'; import { isEqlRule, - isEqlSequenceQuery, isEsqlRule, isNewTermsRule, isThreatMatchRule, @@ -41,7 +40,6 @@ import { THREAT_MATCH_INDEX_HELPER_TEXT, THREAT_MATCH_REQUIRED, THREAT_MATCH_EMPTIES, - EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT, } from './translations'; import { getQueryRequiredMessage } from './utils'; @@ -684,13 +682,6 @@ export const schema: FormSchema = { if (!needsValidation) { return; } - - const query: string = formData.queryBar?.query?.query ?? ''; - if (isEqlSequenceQuery(query)) { - return { - message: EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT, - }; - } }, }, ], diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx index 7d7bb9c4a9253..3e9dcfb7e2f03 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx @@ -218,22 +218,6 @@ export const getEnableThresholdSuppressionLabel = (fields: string[] | undefined) ) ); -export const EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText', - { - defaultMessage: 'Suppression is not supported for EQL sequence queries.', - } -); - -export const EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText', - { - defaultMessage: - '{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Change the EQL query to a non-sequence query, or remove the suppression fields.', - values: { EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP }, - } -); - export const MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.machineLearningSuppressionDisabledLabel', { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 35a1b75aff0fe..d35684b20d766 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -171,9 +171,9 @@ export const eqlExecutor = async ({ } else if (sequences) { console.error('isAlertSuppressionActive??? ', isAlertSuppressionActive); if (isAlertSuppressionActive) { - result.warningMessages.push( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - ); + // result.warningMessages.push( + // 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' + // ); const candidateSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); // partition sequence alert from building block alerts From e44fc04a7c560ca472cccb25fcbf59ef74f7abac Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 12 Aug 2024 15:26:15 -0400 Subject: [PATCH 05/70] update import order, updates logic for get suppressed terms --- .../rule_types/utils/bulk_create_with_suppression.ts | 2 +- .../rule_types/utils/suppression_utils.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 8759c5e5cf372..82603157fd4e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -13,6 +13,7 @@ import type { AlertWithCommonFieldsLatest, SuppressionFieldsLatest, } from '@kbn/rule-registry-plugin/common/schemas'; +import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; import { isQueryRule } from '../../../../../common/detection_engine/utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; @@ -24,7 +25,6 @@ import type { import type { RuleServices } from '../types'; import { createEnrichEventsFunction } from './enrichments'; import type { ExperimentalFeatures } from '../../../../../common'; -import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; import { getNumberOfSuppressedAlerts } from './get_number_of_suppressed_alerts'; export interface GenericBulkCreateResponse { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index d9b6091629021..6506f1abe4e95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -82,10 +82,10 @@ export const getSuppressionTerms = ({ }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - const suppressedProps = pick(fields ?? event?._source, suppressedBy) as Record< - string, - string[] | number[] | undefined - >; + const suppressedProps = pick( + fields != null && fields?.length > 0 ? fields : event?._source, + suppressedBy + ) as Record; const suppressionTerms = suppressedBy.map((field) => { const value = suppressedProps[field] ?? null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; From 68b7863f121dd823fc87a7e3cd2f1c8bd91a589f Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 12 Aug 2024 15:46:01 -0400 Subject: [PATCH 06/70] adds usage of lodash/get function to fetch suppression value --- .../detection_engine/rule_types/utils/suppression_utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 6506f1abe4e95..db92d5f927670 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -8,6 +8,7 @@ import pick from 'lodash/pick'; import get from 'lodash/get'; import sortBy from 'lodash/sortBy'; +import isEmpty from 'lodash/isEmpty'; import { ALERT_SUPPRESSION_DOCS_COUNT, @@ -83,11 +84,11 @@ export const getSuppressionTerms = ({ const suppressedBy = alertSuppression?.groupBy ?? []; const suppressedProps = pick( - fields != null && fields?.length > 0 ? fields : event?._source, + fields != null && !isEmpty(fields) ? fields : event?._source, suppressedBy ) as Record; const suppressionTerms = suppressedBy.map((field) => { - const value = suppressedProps[field] ?? null; + const value = get(suppressedProps, field) ?? null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; return { field, From 837b5ae9608536ec2d33c7a9242b4f476c78745b Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 13 Aug 2024 16:00:28 -0400 Subject: [PATCH 07/70] functionally complete rewrite --- .../create_persistence_rule_type_wrapper.ts | 14 +- .../rule_types/eql/create_eql_alert_type.ts | 26 ++- .../detection_engine/rule_types/eql/eql.ts | 79 +++++---- .../lib/detection_engine/rule_types/types.ts | 3 +- ...bulk_create_suppressed_alerts_in_memory.ts | 141 +++++++++++++++- .../rule_types/utils/suppression_utils.ts | 19 +-- .../utils/wrap_suppressed_alerts.ts | 156 +++++++++++++++++- .../rules/queries/sequence_eql_query.json | 2 +- .../execution_logic/eql_alert_suppression.ts | 75 ++++++++- 9 files changed, 447 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 65b64252b78cb..4fbbdb65c91a0 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -599,12 +599,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); - console.error( - 'WHAT IS AUGMENTED ALERT', - augmentedAlerts.map( - (augAlert) => augAlert._source._source['kibana.alert.group.id'] - ) - ); + // console.error( + // 'WHAT IS AUGMENTED ALERT', + // augmentedAlerts.map( + // (augAlert) => augAlert._source._source['kibana.alert.group.id'] + // ) + // ); const matchingBuildingBlockAlerts = buildingBlockAlerts?.filter((someAlert) => { // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); @@ -612,7 +612,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return ( someAlert?._source[ALERT_GROUP_ID] === - newAlerts[0]?._source._source['kibana.alert.group.id'] + newAlerts[0]?._source['kibana.alert.group.id'] ); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index a8bae32e77260..5190f95832f2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -7,14 +7,18 @@ import { EQL_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; +import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; import { SERVER_APP_ID } from '../../../../../common/constants'; import { EqlRuleParams } from '../../rule_schema'; import { eqlExecutor } from './eql'; -import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types'; +import type { CreateRuleOptions, SecurityAlertType, SignalSource, SignalSourceHit } from '../types'; import { validateIndexPatterns } from '../utils'; import type { BuildReasonMessage } from '../utils/reason_formatters'; -import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; +import { + wrapSuppressedAlerts, + wrapSuppressedSequenceAlerts, +} from '../utils/wrap_suppressed_alerts'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; export const createEqlAlertType = ( @@ -103,6 +107,23 @@ export const createEqlAlertType = ( primaryTimestamp, secondaryTimestamp, }); + const wrapSuppressedSequences = ( + sequences: Array>, + buildReasonMessage: BuildReasonMessage + ) => + wrapSuppressedSequenceAlerts({ + sequences, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + }); const isNonSeqAlertSuppressionActive = await getIsAlertSuppressionActive({ alertSuppression: completeRule.ruleParams.alertSuppression, licensing, @@ -123,6 +144,7 @@ export const createEqlAlertType = ( exceptionFilter, unprocessedExceptions, wrapSuppressedHits, + wrapSuppressedSequences, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive: isNonSeqAlertSuppressionActive, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index d35684b20d766..11420edada552 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -45,7 +45,10 @@ import type { WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; +import { + bulkCreateSuppressedAlertsInMemory, + bulkCreateSuppressedSequencesInMemory, +} from '../utils/bulk_create_suppressed_alerts_in_memory'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; import { ALERT_ANCESTORS, @@ -91,6 +94,7 @@ export const eqlExecutor = async ({ exceptionFilter, unprocessedExceptions, wrapSuppressedHits, + wrapSuppressedSequences, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, @@ -141,13 +145,6 @@ export const eqlExecutor = async ({ const { events, sequences } = response.hits; - console.error('SEQUENCES LENGTH', sequences?.length); - - sequences?.forEach((sequence) => { - console.error('sequence length', sequence.events?.length); - console.error('sequence join keys length', sequence.join_keys?.length); - }); - if (events) { if (isAlertSuppressionActive) { await bulkCreateSuppressedAlertsInMemory({ @@ -175,33 +172,45 @@ export const eqlExecutor = async ({ // 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' // ); - const candidateSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); - // partition sequence alert from building block alerts - const [sequenceAlerts, buildingBlockAlerts] = partition( - candidateSignals, - (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null - ); - console.error( - 'WHAT ARE THE NEW SIGNALS', - sequenceAlerts.map((alert) => alert._source['agent.name']) - ); - await bulkCreateSuppressedAlertsInMemory({ - enrichedEvents: sequenceAlerts, - buildingBlockAlerts, - skipWrapping: true, - toReturn: result, - wrapHits, - bulkCreate, - services, - buildReasonMessage: buildReasonMessageForEqlAlert, - ruleExecutionLogger, - tuple, - alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedHits, - alertTimestampOverride, - alertWithSuppression, - experimentalFeatures, - }); + console.error('how many sequences?', sequences.length); + + /* + We are missing the 'fields' property from the sequences.events because + we are wrapping the sequences before passing them to the bulk create + suppressed function. My hypothesis is to pass the raw sequence data to + bulk create suppressed alerts, then do the sequence wrapping + within the create suppressed alerts function. + + */ + + // commenting out all this code because it needs to happen further down the stack, + // where we are no longer relying on fields. + // const candidateSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); + // // partition sequence alert from building block alerts + // const [sequenceAlerts, buildingBlockAlerts] = partition( + // candidateSignals, + // (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null + // ); + // console.error('how many potential sequence alerts?', sequenceAlerts.length); + try { + await bulkCreateSuppressedSequencesInMemory({ + sequences, + toReturn: result, + wrapSequences, // TODO: fix type mismatch + bulkCreate, + services, + buildReasonMessage: buildReasonMessageForEqlAlert, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedHits: wrapSuppressedSequences, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + }); + } catch (exc) { + console.error('WHAT IS THE EXC', exc); + } } else { newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 381fa76733c0f..4069b7782e0e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -344,8 +344,7 @@ export type WrapHits = ( export type WrapSuppressedHits = ( hits: Array>, - buildReasonMessage: BuildReasonMessage, - skipGenerateId: boolean + buildReasonMessage: BuildReasonMessage ) => Array>; export type WrapSequences = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 86eb38613dbae..4c38693f68f35 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -7,12 +7,17 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; +import { ALERT_BUILDING_BLOCK_TYPE, ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import partition from 'lodash/partition'; + import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType, WrapSuppressedHits, SignalSourceHit, + SignalSource, + WrapSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; import { addToSearchAfterReturn } from './utils'; @@ -57,6 +62,29 @@ export interface BulkCreateSuppressedAlertsParams maxNumberOfAlertsMultiplier?: number; skipWrapping?: boolean; } + +export interface BulkCreateSuppressedSequencesParams + extends Pick< + SearchAfterAndBulkCreateSuppressedAlertsParams, + | 'bulkCreate' + | 'services' + | 'buildReasonMessage' + | 'ruleExecutionLogger' + | 'tuple' + | 'alertSuppression' + | 'wrapSuppressedHits' + | 'alertWithSuppression' + | 'alertTimestampOverride' + > { + wrapSequences: WrapSequences; + sequences: Array>; + buildingBlockAlerts?: Array>; + toReturn: SearchAfterAndBulkCreateReturnType; + experimentalFeatures: ExperimentalFeatures; + mergeSourceAndFields?: boolean; + maxNumberOfAlertsMultiplier?: number; + skipWrapping?: boolean; +} /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, @@ -88,6 +116,11 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ let suppressibleEvents = enrichedEvents; let unsuppressibleWrappedDocs: Array> = []; + // if (suppressibleEvents?.[0]?.events != null) { + // // then we know that we have received sequences + // // so we must augment the wrapHits + // } + if (!suppressOnMissingFields) { const partitionedEvents = partitionMissingFieldsEvents( enrichedEvents, @@ -102,17 +135,16 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ } // refactor the below into a separate function - const suppressibleWrappedDocs = wrapSuppressedHits( - suppressibleEvents, - buildReasonMessage, - skipWrapping - ); + const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage); console.error( 'SUPPRESSIBLE WRAPPED instance ids', suppressibleWrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) ); + // I think we create a separate bulkCreateSuppressedInMemory function + // specifically for eql sequences + // since sequences act differently from the other alerts. return executeBulkCreateAlerts({ suppressibleWrappedDocs, unsuppressibleWrappedDocs, @@ -130,6 +162,103 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ }); }; +/** + * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. + * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, + * regular alerts will be created for such events without suppression + */ +export const bulkCreateSuppressedSequencesInMemory = async ({ + sequences, + toReturn, + wrapSequences, + bulkCreate, + services, + buildReasonMessage, + ruleExecutionLogger, + tuple, + alertSuppression, + wrapSuppressedHits, + alertWithSuppression, + alertTimestampOverride, + experimentalFeatures, + mergeSourceAndFields = false, + maxNumberOfAlertsMultiplier, +}: BulkCreateSuppressedSequencesParams) => { + const suppressOnMissingFields = + (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === + AlertSuppressionMissingFieldsStrategyEnum.suppress; + + let suppressibleSequences: Array> = []; + const unsuppressibleWrappedDocs: Array> = []; + + if (!suppressOnMissingFields) { + sequences.forEach((sequence) => { + // if none of the events in the sequence + // contain a value, then wrap sequence normally, + // otherwise wrap as suppressed + // ask product + const [eventsWithFields, eventsWithoutFields] = partitionMissingFieldsEvents( + sequence.events, + alertSuppression?.groupBy || [], + ['fields'], + mergeSourceAndFields + ); + + if (eventsWithFields.length === 0) { + // unsuppressible sequence alert + unsuppressibleWrappedDocs.push(wrapSequences([sequence], buildReasonMessage)); + console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); + } else { + suppressibleSequences.push(sequence); + } + }); + } else { + suppressibleSequences = sequences; + } + + // refactor the below into a separate function + const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleSequences, buildReasonMessage); + + console.error( + 'SUPPRESSIBLE WRAPPED instance ids', + suppressibleWrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) + ); + + // once we have wrapped thing similarly to + // build alert group from sequence, + // we can pass down the suppressibleWrappeDocs (our sequence alerts) + // and the building block alerts + + // partition sequence alert from building block alerts + const [sequenceAlerts, buildingBlockAlerts] = partition( + suppressibleWrappedDocs, + (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null + ); + + // the code in executeBulkCreateAlerts should + // not have to change, and might even allow me to remove + // some of my earlier changes where the fields property was not available + + // I think we create a separate bulkCreateSuppressedInMemory function + // specifically for eql sequences + // since sequences act differently from the other alerts. + return executeBulkCreateAlerts({ + suppressibleWrappedDocs: sequenceAlerts, + unsuppressibleWrappedDocs, + buildingBlockAlerts, + toReturn, + bulkCreate, + services, + ruleExecutionLogger, + tuple, + alertSuppression, + alertWithSuppression, + alertTimestampOverride, + experimentalFeatures, + maxNumberOfAlertsMultiplier, + }); +}; + export interface ExecuteBulkCreateAlertsParams extends Pick< SearchAfterAndBulkCreateSuppressedAlertsParams, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index db92d5f927670..d2227a0aad12b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -35,24 +35,21 @@ export const getSuppressionAlertFields = ({ suppressionTerms, fallbackTimestamp, instanceId, - event, }: { - fields: Record | undefined; + fields: Record | undefined; primaryTimestamp: string; secondaryTimestamp?: string; suppressionTerms: SuppressionTerm[]; fallbackTimestamp: string; instanceId: string; - event?: SignalSourceHit; }) => { console.error( `primary timestamp: ${primaryTimestamp}, secondaryTimestamp: ${secondaryTimestamp}, fallbackTimestamp: ${fallbackTimestamp}` ); console.error('WHAT ARE FIELDS', JSON.stringify(fields)); - console.error('WHAT ARE EVENT', JSON.stringify(event)); const suppressionTime = new Date( - get(fields ?? event?._source, primaryTimestamp) ?? - (secondaryTimestamp && get(fields ?? event?._source, secondaryTimestamp)) ?? + get(fields, primaryTimestamp) ?? + (secondaryTimestamp && get(fields, secondaryTimestamp)) ?? fallbackTimestamp ); @@ -75,18 +72,16 @@ export const getSuppressionAlertFields = ({ export const getSuppressionTerms = ({ alertSuppression, fields, - event, }: { fields: Record | undefined; alertSuppression: AlertSuppressionCamel | undefined; - event: SignalSourceHit | undefined; }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - const suppressedProps = pick( - fields != null && !isEmpty(fields) ? fields : event?._source, - suppressedBy - ) as Record; + const suppressedProps = pick(fields, suppressedBy) as Record< + string, + string[] | number[] | undefined + >; const suppressionTerms = suppressedBy.map((field) => { const value = get(suppressedProps, field) ?? null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 07af55efa0667..5a31be1e27ebb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -7,12 +7,15 @@ import objectHash from 'object-hash'; -import { TIMESTAMP } from '@kbn/rule-data-utils'; +import { ALERT_BUILDING_BLOCK_TYPE, ALERT_URL, ALERT_UUID, TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import type { SignalSourceHit } from '../types'; +import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; +import type { Ancestor, SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; @@ -26,8 +29,17 @@ import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { buildBulkBody } from '../factories/utils/build_bulk_body'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; import { generateId } from './utils'; +import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids'; import type { BuildReasonMessage } from './reason_formatters'; +import { buildAlertRoot } from '../eql/build_alert_group_from_sequence'; +import { getAlertDetailsUrl } from '@kbn/security-solution-plugin/common/utils/alert_detail_path'; +import { DEFAULT_ALERTS_INDEX } from '@kbn/security-solution-plugin/common/constants'; +import { + ALERT_GROUP_ID, + ALERT_GROUP_INDEX, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { buildAncestors } from '../factories/utils/build_alert'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; @@ -115,3 +127,143 @@ export const wrapSuppressedAlerts = ({ }; }); }; + +/** + * wraps suppressed alerts + * creates instanceId hash, which is used to search on time interval alerts + * populates alert's suppression fields + */ +export const wrapSuppressedSequenceAlerts = ({ + sequences, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, +}: { + sequences: Array>; + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + buildReasonMessage: BuildReasonMessage; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; +}): Array> => { + // objective here is to replicate what is happening + // in x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + // + return sequences.reduce((acc: Array>, sequence) => { + const fields = sequence.events?.reduce( + (seqAcc, event) => ({ ...seqAcc, ...event.fields }), + {} as Record + ); + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields, + }); + const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); + if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { + return []; + } + // The "building block" alerts start out as regular BaseFields. + // We'll add the group ID and index fields + // after creating the shell alert later on + // since that's when the group ID is determined. + const baseAlerts = sequence.events.map((event) => + buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + 'placeholder-alert-uuid', // This is overriden below + publicBaseUrl + ) + ); + + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + console.error('INSTANCE ID', instanceId); + + // The ID of each building block alert depends on all of the other building blocks as well, + // so we generate the IDs after making all the BaseFields + const buildingBlockIds = generateBuildingBlockIds(baseAlerts); + const wrappedBaseFields: Array> = baseAlerts.map( + (block, i): WrappedFieldsLatest => ({ + _id: buildingBlockIds[i], + _index: '', + _source: { + ...block, + [ALERT_UUID]: buildingBlockIds[i], + }, + }) + ); + + // Now that we have an array of building blocks for the events in the sequence, + // we can build the signal that links the building blocks together + // and also insert the group id (which is also the "shell" signal _id) in each building block + const shellAlert = buildAlertRoot( + wrappedBaseFields, + completeRule, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl + ); + const sequenceAlert: WrappedFieldsLatest = { + _id: shellAlert[ALERT_UUID], + _index: '', + _source: { + ...shellAlert, + ...getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + fields, + suppressionTerms, + fallbackTimestamp: baseAlerts?.[0][TIMESTAMP], + instanceId, + }), + }, + }; + + // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks + const wrappedBuildingBlocks = wrappedBaseFields.map( + (block, i): WrappedFieldsLatest => { + const alertUrl = getAlertDetailsUrl({ + alertId: block._id, + index: `${DEFAULT_ALERTS_INDEX}-${spaceId}`, + timestamp: block._source['@timestamp'], + basePath: publicBaseUrl, + spaceId, + }); + + return { + ...block, + _source: { + ...block._source, + [ALERT_BUILDING_BLOCK_TYPE]: 'default', + [ALERT_GROUP_ID]: shellAlert[ALERT_GROUP_ID], + [ALERT_GROUP_INDEX]: i, + [ALERT_URL]: alertUrl, + }, + }; + } + ); + + return [...acc, ...wrappedBuildingBlocks, sequenceAlert]; + }, []); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json index fa49022183d78..c5eda43d78cea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -5,7 +5,7 @@ "https://www.example.com/some-article-about-a-false-positive", "some text string about why another condition could be a false positive" ], - "rule_id": "rule-id-eql-2", + "rule_id": "rule-id-eql-1", "enabled": true, "index": ["auditbeat*", "packetbeat*"], "interval": "30s", diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index 26764650287fc..fc21a95afb799 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -8,6 +8,7 @@ import expect from 'expect'; import { v4 as uuidv4 } from 'uuid'; import sortBy from 'lodash/sortBy'; +import partition from 'lodash/partition'; import { EqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { @@ -48,7 +49,7 @@ import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/ut const getQuery = (id: string) => `any where id == "${id}"`; const getSequenceQuery = (id: string) => - `sequence by id [any where id == "${id}"] [any where id == "${id}"]`; + `sequence [any where id == "${id}"] [any where id == "${id}"]`; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); @@ -1787,6 +1788,78 @@ export default ({ getService }: FtrProviderContext) => { }); describe('sequence queries', () => { + it.skip('only suppresses alerts within the rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + const timestamp1 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + { ...doc1, '@timestamp': timestamp1 }, + // { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect three alerts, two building block and + // one sequence alert, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + console.error(JSON.stringify(sequenceAlert)); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, // suppression ends with later timestamp + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); it('logs a warning if suppression is configured', async () => { const id = uuidv4(); await indexGeneratedSourceDocuments({ From 3f8bb0abfa19525757f21d64fd33972e8d84aa1a Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 3 Sep 2024 16:28:57 -0400 Subject: [PATCH 08/70] mostly console logs, some cleanup --- .../create_persistence_rule_type_wrapper.ts | 33 ++++++---- .../components/step_define_rule/index.tsx | 1 - .../components/step_define_rule/schema.tsx | 15 +---- .../eql/build_alert_group_from_sequence.ts | 2 + .../rule_types/factories/utils/build_alert.ts | 13 ++++ .../factories/utils/build_bulk_body.ts | 1 + ...bulk_create_suppressed_alerts_in_memory.ts | 11 ++-- .../utils/wrap_suppressed_alerts.ts | 6 +- .../rules/queries/sequence_eql_query.json | 10 +-- .../sequence_eql_query_no_duration.json | 61 +++++++++++++++++++ .../execution_logic/eql.ts | 1 + 11 files changed, 113 insertions(+), 41 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index ef94a7a2c3c2f..54bedd56c7eaa 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -27,7 +27,10 @@ import { ALERT_RULE_EXECUTION_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; -import { ALERT_GROUP_ID } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + ALERT_GROUP_ID, + ALERT_ORIGINAL_TIME, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; @@ -86,14 +89,15 @@ const augmentAlerts = ({ }; const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => { - const sourceNoId = (alertSource) => { - if (Object.hasOwn(alertSource, '_id')) { - const { _id, _index, _source, ...source } = alertSource; - return { ..._source, ...source }; - } - return alertSource; - }; - return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, sourceNoId(alert._source)]); //sourceNoId(alert._source)]); + // const sourceNoId = (alertSource) => { + // if (Object.hasOwn(alertSource, '_id')) { + // const { _id, _index, _source, ...source } = alertSource; + // return { ..._source, ...source }; + // } + // return alertSource; + // }; + // return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, sourceNoId(alert._source)]); //sourceNoId(alert._source)]); + return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); }; /** @@ -382,9 +386,13 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper buildingBlockAlerts ) => { try { + // console.error( + // 'ALERT WITH SUPPRESSION INSTANCE IDS', + // alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) + // ); console.error( - 'ALERT WITH SUPPRESSION INSTANCE IDS', - alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) + 'ALERT WITH SUPPRESSION original times', + alerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) ); const ruleDataClientWriter = await ruleDataClient.getWriter({ namespace: options.spaceId, @@ -553,6 +561,9 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, { doc: { + // see if this is where the original time + // is not being set correctly + // when suppressing on per rule exec. ...getUpdatedSuppressionBoundaries( existingAlert, alert, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 28b0ad039b173..d8a90f05de812 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -481,7 +481,6 @@ const StepDefineRuleComponent: FC = ({ /** Suppression fields are generally disabled if either: * - License is insufficient (i.e. less than platinum) - * - An EQL Sequence is used * - ML Field information is not available */ const areSuppressionFieldsDisabled = diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index b80a569bf1754..df7348529c5b5 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -135,7 +135,6 @@ export const schema: FormSchema = { fieldsToValidateOnChange: ['eqlOptions', 'queryBar'], }, queryBar: { - fieldsToValidateOnChange: ['queryBar', 'groupByFields'], validations: [ { validator: ( @@ -657,7 +656,7 @@ export const schema: FormSchema = { ): ReturnType> | undefined => { const [{ formData }] = args; const needsValidation = isSuppressionRuleConfiguredWithGroupBy(formData.ruleType); - + console.error('DO WE NEED VALIDATION', needsValidation); if (!needsValidation) { return; } @@ -672,18 +671,6 @@ export const schema: FormSchema = { })(...args); }, }, - { - validator: ( - ...args: Parameters - ): ReturnType> | undefined => { - const [{ formData, value }] = args; - const groupByLength = (value as string[]).length; - const needsValidation = isEqlRule(formData.ruleType) && groupByLength > 0; - if (!needsValidation) { - return; - } - }, - }, ], }, groupByRadioSelection: {}, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 4e91af23dc91b..7bf6cb1b26113 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -60,6 +60,7 @@ export const buildAlertGroupFromSequence = ( // since that's when the group ID is determined. let baseAlerts: BaseFieldsLatest[] = []; try { + console.error('CALLING WITHIN BUILD ALERT GROUP FROM SEQUENCE'); baseAlerts = sequence.events.map((event) => buildBulkBody( spaceId, @@ -155,6 +156,7 @@ export const buildAlertRoot = ( severity: completeRule.ruleParams.severity, mergedDoc: mergedAlerts as SignalSourceHit, }); + console.error('CALLING INSIDE BUILD ALERT ROOT'); const doc = buildAlert( wrappedBuildingBlocks, completeRule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index d81fe7d020282..f0b633bb0225f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -160,6 +160,7 @@ export const buildAlert = ( riskScoreOverride: number; } ): BaseFieldsLatest => { + // console.error('build alert caller: ', buildAlert.caller); const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce( @@ -191,11 +192,21 @@ export const buildAlert = ( const params = completeRule.ruleParams; + console.error( + 'WHAT ARE DOCS ORIGINAL TIMES before', + docs.map((doc) => doc._source[TIMESTAMP]) + ); + const originalTime = getValidDateFromDoc({ doc: docs[0], primaryTimestamp: TIMESTAMP, }); + console.error( + 'WHAT ARE DOCS ORIGINAL TIMES after', + docs.map((doc) => doc._source[TIMESTAMP]) + ); + const timestamp = alertTimestampOverride?.toISOString() ?? new Date().toISOString(); const alertUrl = getAlertDetailsUrl({ @@ -206,6 +217,8 @@ export const buildAlert = ( spaceId, }); + console.error('WHAT IS ORIGINAL TIME', originalTime?.toISOString()); + return { [TIMESTAMP]: timestamp, [SPACE_IDS]: spaceId != null ? [spaceId] : [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index 9294cc7159c12..93efdc3639b29 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -106,6 +106,7 @@ export const buildBulkBody = ( const thresholdResult = mergedDoc._source?.threshold_result; if (isSourceDoc(mergedDoc)) { + console.error('CALLING INSIDE BUILD BULK BODY'); return { ...validatedSource, ...validatedEventFields, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 4c38693f68f35..5fe1864dd9cef 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -33,6 +33,7 @@ import type { BaseFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -219,11 +220,6 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ // refactor the below into a separate function const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleSequences, buildReasonMessage); - console.error( - 'SUPPRESSIBLE WRAPPED instance ids', - suppressibleWrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) - ); - // once we have wrapped thing similarly to // build alert group from sequence, // we can pass down the suppressibleWrappeDocs (our sequence alerts) @@ -235,6 +231,11 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null ); + console.error( + 'SUPPRESSIBLE WRAPPED original time', + sequenceAlerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) + ); + // the code in executeBulkCreateAlerts should // not have to change, and might even allow me to remove // some of my earlier changes where the fields property was not available diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 5a31be1e27ebb..43bc7702a18b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -108,7 +108,7 @@ export const wrapSuppressedAlerts = ({ } const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - console.error('INSTANCE ID', instanceId); + console.error('suppressed alerts INSTANCE ID', instanceId); return { _id: id, _index: '', @@ -174,6 +174,8 @@ export const wrapSuppressedSequenceAlerts = ({ if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { return []; } + + console.error('CALLING WITHIN WRAP SUPPRESSED SEQUENCE ALERTS'); // The "building block" alerts start out as regular BaseFields. // We'll add the group ID and index fields // after creating the shell alert later on @@ -196,7 +198,7 @@ export const wrapSuppressedSequenceAlerts = ({ ); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - console.error('INSTANCE ID', instanceId); + console.error('sequence alert INSTANCE ID', instanceId); // The ID of each building block alert depends on all of the other building blocks as well, // so we generate the IDs after making all the BaseFields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json index c5eda43d78cea..74b6193c24531 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -1,10 +1,7 @@ { "name": "EQL sequence rule", "description": "Rule with an eql query", - "false_positives": [ - "https://www.example.com/some-article-about-a-false-positive", - "some text string about why another condition could be a false positive" - ], + "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], "rule_id": "rule-id-eql-1", "enabled": true, "index": ["auditbeat*", "packetbeat*"], @@ -55,10 +52,7 @@ ] } ], - "references": [ - "http://www.example.com/some-article-about-attack", - "Some plain text string here explaining why this is a valid thing to look out for" - ], + "references": ["http://www.example.com/some-article-about-attack"], "alert_suppression": { "group_by": ["agent.name"], "duration": { "value": 5, "unit": "h" }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json new file mode 100644 index 0000000000000..32dc6a75d140b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json @@ -0,0 +1,61 @@ +{ + "name": "EQL sequence rule", + "description": "Rule with an eql query", + "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], + "rule_id": "rule-id-eql-1", + "enabled": true, + "index": ["auditbeat*", "packetbeat*"], + "interval": "2m", + "query": "sequence with maxspan=10m [any where agent.type == \"auditbeat\"] [any where event.category == \"network_traffic\"]", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 99, + "to": "now", + "from": "now-5m", + "severity": "high", + "type": "eql", + "language": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": ["http://www.example.com/some-article-about-attack"], + "alert_suppression": { + "group_by": ["agent.name"], + "missing_fields_strategy": "suppress" + }, + "version": 1 +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index d10df654df6aa..f2e5cbb930485 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -510,6 +510,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); + // TODO: investigate original_time here too it('generates shell alerts from EQL sequences in the expected form', async () => { const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['auditbeat-*']), From 13c21bcd08c1c86eed390c1045728702f4965886 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 5 Sep 2024 11:16:53 -0400 Subject: [PATCH 09/70] enables feature flag for ui --- .../common/experimental_features.ts | 5 ++++ .../components/step_define_rule/index.tsx | 16 ++++++++-- ...e_experimental_feature_fields_transform.ts | 30 +++++++++++++++++-- .../pages/rule_editing/index.tsx | 18 +++++++++-- .../logic/use_alert_suppression.tsx | 19 ++++++++++-- .../rule_types/eql/create_eql_alert_type.ts | 4 +-- .../detection_engine/rule_types/eql/eql.ts | 5 +++- .../rule_types/utils/suppression_utils.ts | 2 ++ 8 files changed, 85 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 7d3edafedd1a9..3cc4633236190 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -12,6 +12,11 @@ export type ExperimentalFeatures = { [K in keyof typeof allowedExperimentalValue * This object is then used to validate and parse the value entered. */ export const allowedExperimentalValues = Object.freeze({ + /** + * feature flag for eql sequence alert suppression + */ + alertSuppressionForSequenceEqlRuleEnabled: false, + // FIXME:PT delete? excludePoliciesInFilterEnabled: false, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 8989d866ea672..79e262fd8ec1e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -78,6 +78,7 @@ import { isThresholdRule as getIsThresholdRule, isQueryRule, isEsqlRule, + isEqlSequenceQuery, isSuppressionRuleInGA, } from '../../../../../common/detection_engine/utils'; import { EqlQueryBar } from '../eql_query_bar'; @@ -198,7 +199,6 @@ const StepDefineRuleComponent: FC = ({ }) => { const queryClient = useQueryClient(); - const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); const [threatIndexModified, setThreatIndexModified] = useState(false); @@ -487,6 +487,16 @@ const StepDefineRuleComponent: FC = ({ * purpose and so are treated as if the field is always selected. */ const areSuppressionFieldsSelected = isThresholdRule || groupByFields.length > 0; + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( + ruleType, + isEqlSequenceQuery(queryBar?.query?.query as string) + ); + + useEffect( + () => console.error('IS SUPPRESSION ENABLED', isAlertSuppressionEnabled), + [isAlertSuppressionEnabled] + ); + /** If we don't have ML field information, users can't meaningfully interact with suppression fields */ const areSuppressionFieldsDisabledByMlFields = isMlRule(ruleType) && (mlRuleConfigLoading || !mlSuppressionFields.length); @@ -1187,13 +1197,15 @@ const StepDefineRuleReadOnlyComponent: FC = ({ }) => { const dataForDescription: Partial = getStepDataDataSource(data); const transformFields = useExperimentalFeatureFieldsTransform(); + const fieldsToDisplay = transformFields(dataForDescription); + console.error('WHAT FIELDS TO DISPLAY', fieldsToDisplay.queryBar?.query.query); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index c035fef5af6e4..67221eb865569 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -7,6 +7,8 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { isEqlRule, isEqlSequenceQuery } from '../../../../../common/detection_engine/utils'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -14,9 +16,31 @@ import type { DefineStepRule } from '../../../../detections/pages/detection_engi export const useExperimentalFeatureFieldsTransform = >(): (( fields: T ) => T) => { - const transformer = useCallback((fields: T) => { - return fields; - }, []); + const isAlertSuppressionForSequenceEqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForSequenceEqlRuleEnabled' + ); + const transformer = useCallback( + (fields: T) => { + console.error('fields query bar', fields.queryBar?.query?.query); + const isSuppressionDisabled = + isEqlRule(fields.ruleType) && + isEqlSequenceQuery(fields.queryBar?.query?.query as string) && + !isAlertSuppressionForSequenceEqlRuleEnabled; + + // reset any alert suppression values hidden behind feature flag + if (isSuppressionDisabled) { + return { + ...fields, + groupByFields: [], + groupByRadioSelection: undefined, + groupByDuration: undefined, + suppressionMissingFields: undefined, + }; + } + return fields; + }, + [isAlertSuppressionForSequenceEqlRuleEnabled] + ); return transformer; }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index 1ecbde5b00d7b..a29146d99aa8b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -44,6 +44,7 @@ import { useUserData } from '../../../../detections/components/user_info'; import { StepPanel } from '../../../rule_creation/components/step_panel'; import { StepAboutRule } from '../../components/step_about_rule'; import { StepDefineRule } from '../../components/step_define_rule'; +import { useExperimentalFeatureFieldsTransform } from '../../components/step_define_rule/use_experimental_feature_fields_transform'; import { StepScheduleRule } from '../../components/step_schedule_rule'; import { StepRuleActions } from '../../../rule_creation/components/step_rule_actions'; import { formatRule } from '../rule_creation/helpers'; @@ -54,7 +55,10 @@ import { MaxWidthEuiFlexItem, } from '../../../../detections/pages/detection_engine/rules/helpers'; import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations'; -import { RuleStep } from '../../../../detections/pages/detection_engine/rules/types'; +import { + DefineStepRule, + RuleStep, +} from '../../../../detections/pages/detection_engine/rules/types'; import * as i18n from './translations'; import { SecurityPageName } from '../../../../app/types'; import { ruleStepsOrder } from '../../../../detections/pages/detection_engine/rules/utils'; @@ -392,11 +396,17 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const { startTransaction } = useStartTransaction(); + const defineFieldsTransform = useExperimentalFeatureFieldsTransform(); + const saveChanges = useCallback(async () => { startTransaction({ name: SINGLE_RULE_ACTIONS.SAVE }); + const localDefineStepData: DefineStepRule = defineFieldsTransform({ + ...defineStepForm.getFormData(), + eqlOptions: eqlOptionsSelected, + }); await updateRule({ ...formatRule( - defineStepData, + localDefineStepData, aboutStepData, scheduleStepData, actionsStepData, @@ -414,8 +424,10 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { }, [ aboutStepData, actionsStepData, - defineStepData, + defineStepForm, + defineFieldsTransform, dispatchToaster, + eqlOptionsSelected, navigateToApp, rule?.exceptions_list, rule?.name, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index 6e1b2a4d6163f..f712e6749f50e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -6,20 +6,33 @@ */ import { useCallback } from 'react'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { isEqlRule, isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } -export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { +export const useAlertSuppression = ( + ruleType: Type | undefined, + isEqlSequenceQuery = false +): UseAlertSuppressionReturn => { + const isAlertSuppressionForSequenceEQLRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForSequenceEqlRuleEnabled' + ); + const isSuppressionEnabledForRuleType = useCallback(() => { if (!ruleType) { return false; } + if (isEqlRule(ruleType) && isEqlSequenceQuery) { + console.error('IS SEQUENCE SUPPRESSION ENABLED', isAlertSuppressionForSequenceEQLRuleEnabled); + return isAlertSuppressionForSequenceEQLRuleEnabled; + } + return isSuppressibleAlertRule(ruleType); - }, [ruleType]); + }, [ruleType, isAlertSuppressionForSequenceEQLRuleEnabled, isEqlSequenceQuery]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 5190f95832f2e..ebc788639d82a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -124,7 +124,7 @@ export const createEqlAlertType = ( primaryTimestamp, secondaryTimestamp, }); - const isNonSeqAlertSuppressionActive = await getIsAlertSuppressionActive({ + const isAlertSuppressionActive = await getIsAlertSuppressionActive({ alertSuppression: completeRule.ruleParams.alertSuppression, licensing, }); @@ -147,7 +147,7 @@ export const createEqlAlertType = ( wrapSuppressedSequences, alertTimestampOverride, alertWithSuppression, - isAlertSuppressionActive: isNonSeqAlertSuppressionActive, + isAlertSuppressionActive, experimentalFeatures, }); return { ...result, state }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 11420edada552..d4f4b428d0355 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -167,7 +167,10 @@ export const eqlExecutor = async ({ } } else if (sequences) { console.error('isAlertSuppressionActive??? ', isAlertSuppressionActive); - if (isAlertSuppressionActive) { + if ( + isAlertSuppressionActive && + experimentalFeatures.alertSuppressionForSequenceEqlRuleEnabled + ) { // result.warningMessages.push( // 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' // ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index d2227a0aad12b..d94f46f54edb3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -53,6 +53,8 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp ); + // console.error('SUPPRESSION TIME'); + const suppressionFields = { [ALERT_INSTANCE_ID]: instanceId, [ALERT_SUPPRESSION_TERMS]: suppressionTerms, From d1f726828c30438be56935e8043765827d474ae1 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 9 Sep 2024 11:37:32 -0400 Subject: [PATCH 10/70] type fixes --- .../create_persistence_rule_type_wrapper.ts | 603 +++++++++--------- .../server/utils/persistence_types.ts | 2 +- .../model/alerts/8.0.0/index.ts | 16 +- .../components/step_define_rule/schema.tsx | 1 - .../rule_types/eql/create_eql_alert_type.ts | 4 +- .../rule_types/eql/eql.test.ts | 4 + .../detection_engine/rule_types/eql/eql.ts | 10 +- .../lib/detection_engine/rule_types/types.ts | 5 + ...bulk_create_suppressed_alerts_in_memory.ts | 17 +- .../utils/bulk_create_with_suppression.ts | 23 +- .../rule_types/utils/suppression_utils.ts | 12 +- .../utils/wrap_suppressed_alerts.ts | 215 +++---- 12 files changed, 477 insertions(+), 435 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 54bedd56c7eaa..c89c3f4fdcea7 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -27,15 +27,19 @@ import { ALERT_RULE_EXECUTION_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; -import { - ALERT_GROUP_ID, - ALERT_ORIGINAL_TIME, -} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +// import { +// ALERT_GROUP_ID, +// ALERT_ORIGINAL_TIME, +// } from '@kbn/security-solution-plugin/common/field_maps/field_names'; import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; import { errorAggregator } from './utils'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; +// import { +// isEqlBuildingBlockAlert, +// isEqlShellAlert, +// } from '@kbn/security-solution-plugin/common/api/detection_engine/model/alerts/8.0.0'; /** * Alerts returned from BE have date type coerced to ISO strings @@ -65,10 +69,10 @@ const augmentAlerts = ({ kibanaVersion: string; currentTimeOverride: Date | undefined; }) => { - console.error( - 'DO AUGMENTED ALERTS HAVE INSTANCE IDS', - alerts.map((alert) => alert._source[ALERT_INSTANCE_ID]) - ); + // console.error( + // 'DO AUGMENTED ALERTS HAVE INSTANCE IDS', + // alerts.map((alert) => alert._source[ALERT_INSTANCE_ID]) + // ); const commonRuleFields = getCommonAlertFields(options); return alerts.map((alert) => { return { @@ -383,327 +387,282 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, isRuleExecutionOnly, maxAlerts, - buildingBlockAlerts + getMatchingBuildingBlockAlerts ) => { - try { - // console.error( - // 'ALERT WITH SUPPRESSION INSTANCE IDS', - // alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) - // ); - console.error( - 'ALERT WITH SUPPRESSION original times', - alerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) - ); - const ruleDataClientWriter = await ruleDataClient.getWriter({ - namespace: options.spaceId, - }); + // console.error( + // 'ALERT WITH SUPPRESSION INSTANCE IDS', + // alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) + // ); + // console.error( + // 'ALERT WITH SUPPRESSION original times', + // alerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) + // ); + const ruleDataClientWriter = await ruleDataClient.getWriter({ + namespace: options.spaceId, + }); - // Only write alerts if: - // - writing is enabled - // AND - // - rule execution has not been cancelled due to timeout - // OR - // - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway - const writeAlerts = - ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); + // Only write alerts if: + // - writing is enabled + // AND + // - rule execution has not been cancelled due to timeout + // OR + // - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway + const writeAlerts = + ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts(); - let alertsWereTruncated = false; + let alertsWereTruncated = false; - if (writeAlerts && alerts.length > 0) { - const suppressionWindowStart = dateMath.parse(suppressionWindow, { - forceNow: currentTimeOverride, - }); + if (writeAlerts && alerts.length > 0) { + const suppressionWindowStart = dateMath.parse(suppressionWindow, { + forceNow: currentTimeOverride, + }); - if (!suppressionWindowStart) { - throw new Error('Failed to parse suppression window'); - } + if (!suppressionWindowStart) { + throw new Error('Failed to parse suppression window'); + } - const filteredDuplicates = await filterDuplicateAlerts({ - alerts, - ruleDataClient, - spaceId: options.spaceId, - }); + const filteredDuplicates = await filterDuplicateAlerts({ + alerts, + ruleDataClient, + spaceId: options.spaceId, + }); - if (filteredDuplicates.length === 0) { - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated, - }; - } + if (filteredDuplicates.length === 0) { + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated, + }; + } - const suppressionAlertSearchRequest = { - body: { - size: filteredDuplicates.length, - query: { - bool: { - filter: [ - { - range: { - [ALERT_START]: { - gte: suppressionWindowStart.toISOString(), - }, + const suppressionAlertSearchRequest = { + body: { + size: filteredDuplicates.length, + query: { + bool: { + filter: [ + { + range: { + [ALERT_START]: { + gte: suppressionWindowStart.toISOString(), }, }, - { - terms: { - [ALERT_INSTANCE_ID]: filteredDuplicates.map( - (alert) => alert._source[ALERT_INSTANCE_ID] - ), - }, + }, + { + terms: { + [ALERT_INSTANCE_ID]: filteredDuplicates.map( + (alert) => alert._source[ALERT_INSTANCE_ID] + ), }, - { - bool: { - must_not: { - term: { - [ALERT_WORKFLOW_STATUS]: 'closed', - }, + }, + { + bool: { + must_not: { + term: { + [ALERT_WORKFLOW_STATUS]: 'closed', }, }, }, - ], - }, - }, - collapse: { - field: ALERT_INSTANCE_ID, - }, - sort: [ - { - [ALERT_START]: { - order: 'desc' as const, }, - }, - ], + ], + }, }, + collapse: { + field: ALERT_INSTANCE_ID, + }, + sort: [ + { + [ALERT_START]: { + order: 'desc' as const, + }, + }, + ], + }, + }; + + const response = await ruleDataClient + .getReader({ namespace: options.spaceId }) + .search< + typeof suppressionAlertSearchRequest, + BackendAlertWithSuppressionFields870<{}> + >(suppressionAlertSearchRequest); + + const existingAlertsByInstanceId = response.hits.hits.reduce< + Record>> + >((acc, hit) => { + acc[hit._source[ALERT_INSTANCE_ID]] = hit; + return acc; + }, {}); + + // filter out alerts that were already suppressed + // alert was suppressed if its suppression ends is older + // than suppression end of existing alert + // if existing alert was created earlier during the same + // rule execution - then alerts can be counted as not suppressed yet + // as they are processed for the first time against this existing alert + const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + if ( + !existingAlert || + existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId + ) { + return true; + } + + return !isExistingDateGtEqThanAlert(existingAlert, alert, ALERT_SUPPRESSION_END); + }); + + if (nonSuppressedAlerts.length === 0) { + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated, }; + } + + const { alertCandidates, suppressedAlerts: suppressedInMemoryAlerts } = + suppressAlertsInMemory(nonSuppressedAlerts); - const response = await ruleDataClient - .getReader({ namespace: options.spaceId }) - .search< - typeof suppressionAlertSearchRequest, - BackendAlertWithSuppressionFields870<{}> - >(suppressionAlertSearchRequest); - - const existingAlertsByInstanceId = response.hits.hits.reduce< - Record>> - >((acc, hit) => { - acc[hit._source[ALERT_INSTANCE_ID]] = hit; - return acc; - }, {}); - - // filter out alerts that were already suppressed - // alert was suppressed if its suppression ends is older - // than suppression end of existing alert - // if existing alert was created earlier during the same - // rule execution - then alerts can be counted as not suppressed yet - // as they are processed for the first time against this existing alert - const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - - if ( - !existingAlert || + const [duplicateAlerts, newAlerts] = partition(alertCandidates, (alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + + // if suppression enabled only on rule execution, we need to suppress alerts only against + // alert created in the same rule execution. Otherwise, we need to create a new alert to accommodate per rule execution suppression + if (isRuleExecutionOnly) { + return ( existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId - ) { - return true; - } - - return !isExistingDateGtEqThanAlert( - existingAlert, - alert, - ALERT_SUPPRESSION_END ); - }); - - if (nonSuppressedAlerts.length === 0) { - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated, - }; + } else { + return existingAlert != null; } + }); - const { alertCandidates, suppressedAlerts: suppressedInMemoryAlerts } = - suppressAlertsInMemory(nonSuppressedAlerts); - - const [duplicateAlerts, newAlerts] = partition(alertCandidates, (alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - - // if suppression enabled only on rule execution, we need to suppress alerts only against - // alert created in the same rule execution. Otherwise, we need to create a new alert to accommodate per rule execution suppression - if (isRuleExecutionOnly) { - return ( - existingAlert?._source?.[ALERT_RULE_EXECUTION_UUID] === options.executionId - ); - } else { - return existingAlert != null; - } - }); - - console.error('ARE THERE DUPLICATE ALERTS', duplicateAlerts?.length); - - const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { - const existingAlert = - existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; - const existingDocsCount = - existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; - - return [ - { - update: { - _id: existingAlert._id, - _index: existingAlert._index, - require_alias: false, - }, + console.error('ARE THERE DUPLICATE ALERTS', duplicateAlerts?.length); + + const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { + const existingAlert = + existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; + const existingDocsCount = + existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0; + + return [ + { + update: { + _id: existingAlert._id, + _index: existingAlert._index, + require_alias: false, }, - { - doc: { - // see if this is where the original time - // is not being set correctly - // when suppressing on per rule exec. - ...getUpdatedSuppressionBoundaries( - existingAlert, - alert, - options.executionId - ), - [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), - [ALERT_SUPPRESSION_DOCS_COUNT]: - existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, - }, + }, + { + doc: { + // see if this is where the original time + // is not being set correctly + // when suppressing on per rule exec. + ...getUpdatedSuppressionBoundaries( + existingAlert, + alert, + options.executionId + ), + [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), + [ALERT_SUPPRESSION_DOCS_COUNT]: + existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1, }, - ]; - }); - - let enrichedAlerts = newAlerts; - console.error('WHAT ARE ENRICHED ALERTS LENGTH BEFORE', enrichedAlerts?.length); - - if (enrichAlerts) { - try { - enrichedAlerts = await enrichAlerts(enrichedAlerts, { - spaceId: options.spaceId, - }); - } catch (e) { - logger.debug('Enrichments failed'); - } - } + }, + ]; + }); - console.error('WHAT ARE ENRICHED ALERTS LENGTH AFTER', enrichedAlerts?.length); + let enrichedAlerts = newAlerts; + console.error('WHAT ARE ENRICHED ALERTS LENGTH BEFORE', enrichedAlerts?.length); - if (maxAlerts && enrichedAlerts.length > maxAlerts) { - enrichedAlerts.length = maxAlerts; - alertsWereTruncated = true; + if (enrichAlerts) { + try { + enrichedAlerts = await enrichAlerts(enrichedAlerts, { + spaceId: options.spaceId, + }); + } catch (e) { + logger.debug('Enrichments failed'); } + } - const augmentedAlerts = augmentAlerts({ - alerts: enrichedAlerts, - options, - kibanaVersion: ruleDataClient.kibanaVersion, - currentTimeOverride, - }); + console.error('WHAT ARE ENRICHED ALERTS LENGTH AFTER', enrichedAlerts?.length); - // console.error( - // 'WHAT IS AUGMENTED ALERT', - // augmentedAlerts.map( - // (augAlert) => augAlert._source._source['kibana.alert.group.id'] - // ) - // ); + if (maxAlerts && enrichedAlerts.length > maxAlerts) { + enrichedAlerts.length = maxAlerts; + alertsWereTruncated = true; + } - const matchingBuildingBlockAlerts = buildingBlockAlerts?.filter((someAlert) => { - // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); - // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); + const augmentedAlerts = augmentAlerts({ + alerts: enrichedAlerts, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + currentTimeOverride, + }); - return ( - someAlert?._source[ALERT_GROUP_ID] === - newAlerts[0]?._source['kibana.alert.group.id'] - ); - }); - - const augmentedBuildingBlockAlerts = - newAlerts?.length > 0 && buildingBlockAlerts?.length > 0 - ? augmentAlerts({ - alerts: matchingBuildingBlockAlerts, - options, - kibanaVersion: ruleDataClient.kibanaVersion, - currentTimeOverride, - }) - : []; - - // console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); - // console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); - // console.error( - // 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', - // JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) - // ); - const bulkResponse = await ruleDataClientWriter.bulk({ - body: [ - ...duplicateAlertUpdates, - ...mapAlertsToBulkCreate(augmentedAlerts), - ...(newAlerts?.length > 0 - ? mapAlertsToBulkCreate(augmentedBuildingBlockAlerts) - : []), - ], - refresh: true, - }); + // console.error( + // 'WHAT IS AUGMENTED ALERT', + // augmentedAlerts.map( + // (augAlert) => augAlert._source._source['kibana.alert.group.id'] + // ) + // ); - if (bulkResponse == null) { - return { - createdAlerts: [], - errors: {}, - suppressedAlerts: [], - alertsWereTruncated: false, - }; - } + /** + * buildingBlockAlerts?.filter((someAlert) => { + // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); + // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); - const createdAlerts = augmentedAlerts - .map((alert, idx) => { - const responseItem = - bulkResponse.body.items[idx + duplicateAlerts.length].create; - return { - _id: responseItem?._id ?? '', - _index: responseItem?._index ?? '', - ...alert._source, - }; - }) - .filter( - (_, idx) => - bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 - ) - // Security solution's EQL rule consists of building block alerts which should be filtered out. - // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. - .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); - - createdAlerts.forEach((alert) => - options.services.alertFactory - .create(alert._id) - .replaceState({ - signals_count: 1, - }) - .scheduleActions(type.defaultActionGroupId, { - rule: mapKeys(snakeCase, { - ...options.params, - name: options.rule.name, - id: options.rule.id, - }), - results_link: type.getViewInAppRelativeUrl?.({ - rule: { ...options.rule, params: options.params }, - start: Date.parse(alert[TIMESTAMP]), - end: Date.parse(alert[TIMESTAMP]), - }), - alerts: [formatAlert?.(alert) ?? alert], - }) + return ( + isEqlBuildingBlockAlert(someAlert?._source) && + isEqlShellAlert(newAlerts?.[0]?._source) && + someAlert?._source?.[ALERT_GROUP_ID] === newAlerts[0]?._source[ALERT_GROUP_ID] ); + }); + */ + + console.error('is it defined', getMatchingBuildingBlockAlerts); + + // TODO: move all this stuff into a utility function + // only create building block alerts for the new eql shell alert + const matchingBuildingBlockAlerts = + newAlerts?.length > 0 && getMatchingBuildingBlockAlerts != null + ? getMatchingBuildingBlockAlerts(newAlerts[0]?._source) + : []; + + const augmentedBuildingBlockAlerts = + getMatchingBuildingBlockAlerts != null && newAlerts?.length > 0 + ? augmentAlerts({ + alerts: matchingBuildingBlockAlerts as unknown as Array<{ + _id: string; + _source: unknown; + }>, + options, + kibanaVersion: ruleDataClient.kibanaVersion, + currentTimeOverride, + }) + : []; - return { - createdAlerts, - suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts], - errors: errorAggregator(bulkResponse.body, [409]), - alertsWereTruncated, - }; - } else { - logger.debug('Writing is disabled.'); + // console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); + // console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); + // console.error( + // 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', + // JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) + // ); + const bulkResponse = await ruleDataClientWriter.bulk({ + body: [ + ...duplicateAlertUpdates, + ...mapAlertsToBulkCreate(augmentedAlerts), + ...(newAlerts?.length > 0 + ? mapAlertsToBulkCreate(augmentedBuildingBlockAlerts) + : []), + ], + refresh: true, + }); + + if (bulkResponse == null) { return { createdAlerts: [], errors: {}, @@ -711,8 +670,60 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alertsWereTruncated: false, }; } - } catch (exc) { - console.error('BIG TIME EXCEPTION', exc); + + const createdAlerts = augmentedAlerts + .map((alert, idx) => { + const responseItem = + bulkResponse.body.items[idx + duplicateAlerts.length].create; + return { + _id: responseItem?._id ?? '', + _index: responseItem?._index ?? '', + ...alert._source, + }; + }) + .filter( + (_, idx) => + bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 + ) + // Security solution's EQL rule consists of building block alerts which should be filtered out. + // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. + .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); + + createdAlerts.forEach((alert) => + options.services.alertFactory + .create(alert._id) + .replaceState({ + signals_count: 1, + }) + .scheduleActions(type.defaultActionGroupId, { + rule: mapKeys(snakeCase, { + ...options.params, + name: options.rule.name, + id: options.rule.id, + }), + results_link: type.getViewInAppRelativeUrl?.({ + rule: { ...options.rule, params: options.params }, + start: Date.parse(alert[TIMESTAMP]), + end: Date.parse(alert[TIMESTAMP]), + }), + alerts: [formatAlert?.(alert) ?? alert], + }) + ); + + return { + createdAlerts, + suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts], + errors: errorAggregator(bulkResponse.body, [409]), + alertsWereTruncated, + }; + } else { + logger.debug('Writing is disabled.'); + return { + createdAlerts: [], + errors: {}, + suppressedAlerts: [], + alertsWereTruncated: false, + }; } }, }, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 0e9ac1b3beb41..b23efde7095fe 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -54,7 +54,7 @@ export type SuppressedAlertService = ( currentTimeOverride?: Date, isRuleExecutionOnly?: boolean, maxAlerts?: number, - buildingBlockAlerts?: Array<{ _id: string; _source: T }> + getMatchingBuildingBlockAlerts?: (alert: unknown) => Array ) => Promise>; export interface SuppressedAlertServiceResult extends PersistenceAlertServiceResult { diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts index 3b7237ae8bb71..24ed99b9386c9 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts @@ -6,7 +6,6 @@ */ import type { - ALERT_BUILDING_BLOCK_TYPE, ALERT_REASON, ALERT_RISK_SCORE, ALERT_RULE_AUTHOR, @@ -34,12 +33,14 @@ import type { ALERT_RULE_VERSION, ALERT_SEVERITY, ALERT_STATUS, - ALERT_UUID, ALERT_WORKFLOW_STATUS, EVENT_KIND, SPACE_IDS, TIMESTAMP, } from '@kbn/rule-data-utils'; + +import { ALERT_BUILDING_BLOCK_TYPE, ALERT_UUID } from '@kbn/rule-data-utils'; + // TODO: Create and import 8.0.0 versioned ListArray schema import type { ListArray } from '@kbn/securitysolution-io-ts-list-types'; // TODO: Create and import 8.0.0 versioned alerting-types schemas @@ -57,7 +58,6 @@ import type { ALERT_RULE_ACTIONS, ALERT_RULE_EXCEPTIONS_LIST, ALERT_RULE_FALSE_POSITIVES, - ALERT_GROUP_ID, ALERT_GROUP_INDEX, ALERT_RULE_IMMUTABLE, ALERT_RULE_MAX_SIGNALS, @@ -69,6 +69,7 @@ import type { ALERT_RULE_TIMELINE_TITLE, ALERT_RULE_TIMESTAMP_OVERRIDE, } from '../../../../../field_maps/field_names'; +import { ALERT_GROUP_ID } from '../../../../../field_maps/field_names'; // TODO: Create and import 8.0.0 versioned RuleAlertAction type import type { SearchTypes } from '../../../../../detection_engine/types'; import type { RuleAction } from '../../rule_schema'; @@ -163,8 +164,17 @@ export interface EqlShellFields800 extends BaseFields800 { export type EqlBuildingBlockAlert800 = AlertWithCommonFields800; +export const isEqlBuildingBlockAlert = ( + alertObject: unknown +): alertObject is EqlBuildingBlockAlert800 => + (alertObject as EqlBuildingBlockAlert800)?.[ALERT_BUILDING_BLOCK_TYPE] != null; + export type EqlShellAlert800 = AlertWithCommonFields800; +export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAlert800 => + (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && + (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; + export type GenericAlert800 = AlertWithCommonFields800; // This is the type of the final generated alert including base fields, common fields diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index df7348529c5b5..efc08a9b6c839 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -17,7 +17,6 @@ import { customValidators, } from '../../../../common/components/threat_match/helpers'; import { - isEqlRule, isEsqlRule, isNewTermsRule, isThreatMatchRule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index ebc788639d82a..00ecde956776b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -90,8 +90,7 @@ export const createEqlAlertType = ( const wrapSuppressedHits = ( events: SignalSourceHit[], - buildReasonMessage: BuildReasonMessage, - skipGenerateId: boolean + buildReasonMessage: BuildReasonMessage ) => wrapSuppressedAlerts({ events, @@ -103,7 +102,6 @@ export const createEqlAlertType = ( alertTimestampOverride, ruleExecutionLogger, publicBaseUrl, - skipGenerateId, primaryTimestamp, secondaryTimestamp, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index c16d61d3b0ea5..01406cee3a209 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -64,6 +64,7 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), + wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [getExceptionListItemSchemaMock()], @@ -113,6 +114,7 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), + wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], @@ -146,6 +148,7 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), + wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], @@ -181,6 +184,7 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), + wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index d4f4b428d0355..7a23beb2ffe07 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -5,7 +5,6 @@ * 2.0. */ import { performance } from 'perf_hooks'; -import { partition } from 'lodash'; import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; @@ -28,6 +27,7 @@ import type { SearchAfterAndBulkCreateReturnType, SignalSource, WrapSuppressedHits, + WrapSuppressedSequences, } from '../types'; import { addToSearchAfterReturn, @@ -50,11 +50,6 @@ import { bulkCreateSuppressedSequencesInMemory, } from '../utils/bulk_create_suppressed_alerts_in_memory'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; -import { - ALERT_ANCESTORS, - ALERT_BUILDING_BLOCK_TYPE, - ALERT_GROUP_ID, -} from '@kbn/security-solution-plugin/common/field_maps/field_names'; interface EqlExecutorParams { inputIndex: string[]; @@ -72,6 +67,7 @@ interface EqlExecutorParams { exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; wrapSuppressedHits: WrapSuppressedHits; + wrapSuppressedSequences: WrapSuppressedSequences; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; isAlertSuppressionActive: boolean; @@ -206,7 +202,7 @@ export const eqlExecutor = async ({ ruleExecutionLogger, tuple, alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedHits: wrapSuppressedSequences, + wrapSuppressedSequences, alertTimestampOverride, alertWithSuppression, experimentalFeatures, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 4069b7782e0e8..f4d37b4b1d817 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -347,6 +347,11 @@ export type WrapSuppressedHits = ( buildReasonMessage: BuildReasonMessage ) => Array>; +export type WrapSuppressedSequences = ( + sequences: Array>, + buildReasonMessage: BuildReasonMessage +) => Array>; + export type WrapSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 5fe1864dd9cef..227c5cf213b95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -18,6 +18,7 @@ import type { SignalSourceHit, SignalSource, WrapSequences, + WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; import { addToSearchAfterReturn } from './utils'; @@ -37,6 +38,7 @@ import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_ interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; + wrapSuppressedSequences: WrapSuppressedSequences; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; alertSuppression?: AlertSuppressionCamel; @@ -73,7 +75,7 @@ export interface BulkCreateSuppressedSequencesParams | 'ruleExecutionLogger' | 'tuple' | 'alertSuppression' - | 'wrapSuppressedHits' + | 'wrapSuppressedSequences' | 'alertWithSuppression' | 'alertTimestampOverride' > { @@ -178,7 +180,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ ruleExecutionLogger, tuple, alertSuppression, - wrapSuppressedHits, + wrapSuppressedSequences, alertWithSuppression, alertTimestampOverride, experimentalFeatures, @@ -190,7 +192,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ AlertSuppressionMissingFieldsStrategyEnum.suppress; let suppressibleSequences: Array> = []; - const unsuppressibleWrappedDocs: Array> = []; + const unsuppressibleWrappedDocs: Array> = []; if (!suppressOnMissingFields) { sequences.forEach((sequence) => { @@ -198,7 +200,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ // contain a value, then wrap sequence normally, // otherwise wrap as suppressed // ask product - const [eventsWithFields, eventsWithoutFields] = partitionMissingFieldsEvents( + const [eventsWithFields] = partitionMissingFieldsEvents( sequence.events, alertSuppression?.groupBy || [], ['fields'], @@ -207,7 +209,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ if (eventsWithFields.length === 0) { // unsuppressible sequence alert - unsuppressibleWrappedDocs.push(wrapSequences([sequence], buildReasonMessage)); + unsuppressibleWrappedDocs.concat(...wrapSequences([sequence], buildReasonMessage)); console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); } else { suppressibleSequences.push(sequence); @@ -218,7 +220,10 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ } // refactor the below into a separate function - const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleSequences, buildReasonMessage); + const suppressibleWrappedDocs = wrapSuppressedSequences( + suppressibleSequences, + buildReasonMessage + ); // once we have wrapped thing similarly to // build alert group from sequence, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 82603157fd4e0..a398ba5683341 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -26,6 +26,11 @@ import type { RuleServices } from '../types'; import { createEnrichEventsFunction } from './enrichments'; import type { ExperimentalFeatures } from '../../../../../common'; import { getNumberOfSuppressedAlerts } from './get_number_of_suppressed_alerts'; +import { + isEqlBuildingBlockAlert, + isEqlShellAlert, +} from '@kbn/security-solution-plugin/common/api/detection_engine/model/alerts/8.0.0'; +import { ALERT_GROUP_ID } from '@kbn/security-solution-plugin/common/field_maps/field_names'; export interface GenericBulkCreateResponse { success: boolean; @@ -105,6 +110,22 @@ export const bulkCreateWithSuppression = async < wrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) ); + let myfunc; + + if (buildingBlockAlerts != null && buildingBlockAlerts.length > 0) + myfunc = (newAlertSource: unknown) => { + return buildingBlockAlerts?.filter((someAlert) => { + // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); + // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); + + return ( + isEqlBuildingBlockAlert(newAlertSource) && + isEqlShellAlert(newAlertSource) && + someAlert?._source?.[ALERT_GROUP_ID] === newAlertSource?.[ALERT_GROUP_ID] + ); + }); + }; + const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( wrappedDocs.map((doc) => ({ @@ -118,7 +139,7 @@ export const bulkCreateWithSuppression = async < alertTimestampOverride, isSuppressionPerRuleExecution, maxAlerts, - buildingBlockAlerts // do the same map as wrappedDocs + myfunc // do the same map as wrappedDocs ); const end = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index d94f46f54edb3..fbb3ff2c853e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -8,7 +8,6 @@ import pick from 'lodash/pick'; import get from 'lodash/get'; import sortBy from 'lodash/sortBy'; -import isEmpty from 'lodash/isEmpty'; import { ALERT_SUPPRESSION_DOCS_COUNT, @@ -18,7 +17,6 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; -import { SignalSourceHit } from '../types'; export interface SuppressionTerm { field: string; @@ -47,11 +45,11 @@ export const getSuppressionAlertFields = ({ `primary timestamp: ${primaryTimestamp}, secondaryTimestamp: ${secondaryTimestamp}, fallbackTimestamp: ${fallbackTimestamp}` ); console.error('WHAT ARE FIELDS', JSON.stringify(fields)); - const suppressionTime = new Date( - get(fields, primaryTimestamp) ?? - (secondaryTimestamp && get(fields, secondaryTimestamp)) ?? - fallbackTimestamp - ); + const primeTimestamp = get(fields, primaryTimestamp) as string; + const secondTimestamp = + secondaryTimestamp != null ? (get(fields, secondaryTimestamp) as string) : fallbackTimestamp; + + const suppressionTime = new Date(primeTimestamp ?? secondTimestamp); // console.error('SUPPRESSION TIME'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 43bc7702a18b4..51fd242febbfb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -14,8 +14,6 @@ import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; import type { Ancestor, SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, - EqlBuildingBlockFieldsLatest, - EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; @@ -59,7 +57,6 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger, publicBaseUrl, primaryTimestamp, - skipGenerateId = false, secondaryTimestamp, }: { events: SignalSourceHit[]; @@ -72,43 +69,39 @@ export const wrapSuppressedAlerts = ({ ruleExecutionLogger: IRuleExecutionLogForExecutors; publicBaseUrl: string | undefined; primaryTimestamp: string; - skipGenerateId: boolean; secondaryTimestamp?: string; }): Array> => { return events.map((event) => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, fields: event.fields, - event, }); - let id = event._id; - let baseAlert: BaseFieldsLatest = event; - if (!skipGenerateId) { - id = generateId( - event._index, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - event._id!, - String(event._version), - `${spaceId}:${completeRule.alertId}` - ); - baseAlert = buildBulkBody( - spaceId, - completeRule, - event, - mergeStrategy, - [], - true, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - ruleExecutionLogger, - id, - publicBaseUrl - ); - } + + const id = generateId( + event._index, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + event._id!, + String(event._version), + `${spaceId}:${completeRule.alertId}` + ); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - console.error('suppressed alerts INSTANCE ID', instanceId); + + const baseAlert: BaseFieldsLatest = buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + return { _id: id, _index: '', @@ -118,7 +111,6 @@ export const wrapSuppressedAlerts = ({ primaryTimestamp, secondaryTimestamp, fields: event.fields, - event, suppressionTerms, fallbackTimestamp: baseAlert[TIMESTAMP], instanceId, @@ -161,90 +153,90 @@ export const wrapSuppressedSequenceAlerts = ({ // objective here is to replicate what is happening // in x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts // - return sequences.reduce((acc: Array>, sequence) => { - const fields = sequence.events?.reduce( - (seqAcc, event) => ({ ...seqAcc, ...event.fields }), - {} as Record - ); - const suppressionTerms = getSuppressionTerms({ - alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields, - }); - const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); - if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { - return []; - } + return sequences.reduce( + (acc: Array>, sequence) => { + const fields = sequence.events?.reduce( + (seqAcc, event) => ({ ...seqAcc, ...event.fields }), + {} as Record + ); + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields, + }); + const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); + if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { + return []; + } - console.error('CALLING WITHIN WRAP SUPPRESSED SEQUENCE ALERTS'); - // The "building block" alerts start out as regular BaseFields. - // We'll add the group ID and index fields - // after creating the shell alert later on - // since that's when the group ID is determined. - const baseAlerts = sequence.events.map((event) => - buildBulkBody( - spaceId, + console.error('CALLING WITHIN WRAP SUPPRESSED SEQUENCE ALERTS'); + // The "building block" alerts start out as regular BaseFields. + // We'll add the group ID and index fields + // after creating the shell alert later on + // since that's when the group ID is determined. + const baseAlerts = sequence.events.map((event) => + buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + 'placeholder-alert-uuid', // This is overriden below + publicBaseUrl + ) + ); + + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + console.error('sequence alert INSTANCE ID', instanceId); + + // The ID of each building block alert depends on all of the other building blocks as well, + // so we generate the IDs after making all the BaseFields + const buildingBlockIds = generateBuildingBlockIds(baseAlerts); + const wrappedBaseFields: Array> = baseAlerts.map( + (block, i): WrappedFieldsLatest => ({ + _id: buildingBlockIds[i], + _index: '', + _source: { + ...block, + [ALERT_UUID]: buildingBlockIds[i], + }, + }) + ); + + // Now that we have an array of building blocks for the events in the sequence, + // we can build the signal that links the building blocks together + // and also insert the group id (which is also the "shell" signal _id) in each building block + const shellAlert = buildAlertRoot( + wrappedBaseFields, completeRule, - event, - mergeStrategy, - [], - true, + spaceId, buildReasonMessage, indicesToQuery, alertTimestampOverride, - ruleExecutionLogger, - 'placeholder-alert-uuid', // This is overriden below publicBaseUrl - ) - ); - - const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - console.error('sequence alert INSTANCE ID', instanceId); - - // The ID of each building block alert depends on all of the other building blocks as well, - // so we generate the IDs after making all the BaseFields - const buildingBlockIds = generateBuildingBlockIds(baseAlerts); - const wrappedBaseFields: Array> = baseAlerts.map( - (block, i): WrappedFieldsLatest => ({ - _id: buildingBlockIds[i], + ); + const sequenceAlert = { + _id: shellAlert[ALERT_UUID], _index: '', _source: { - ...block, - [ALERT_UUID]: buildingBlockIds[i], + ...shellAlert, + ...getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + fields, + suppressionTerms, + fallbackTimestamp: baseAlerts?.[0][TIMESTAMP], + instanceId, + }), }, - }) - ); + }; - // Now that we have an array of building blocks for the events in the sequence, - // we can build the signal that links the building blocks together - // and also insert the group id (which is also the "shell" signal _id) in each building block - const shellAlert = buildAlertRoot( - wrappedBaseFields, - completeRule, - spaceId, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - publicBaseUrl - ); - const sequenceAlert: WrappedFieldsLatest = { - _id: shellAlert[ALERT_UUID], - _index: '', - _source: { - ...shellAlert, - ...getSuppressionAlertFields({ - primaryTimestamp, - secondaryTimestamp, - fields, - suppressionTerms, - fallbackTimestamp: baseAlerts?.[0][TIMESTAMP], - instanceId, - }), - }, - }; - - // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks - const wrappedBuildingBlocks = wrappedBaseFields.map( - (block, i): WrappedFieldsLatest => { + // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks + const wrappedBuildingBlocks = wrappedBaseFields.map((block, i) => { const alertUrl = getAlertDetailsUrl({ alertId: block._id, index: `${DEFAULT_ALERTS_INDEX}-${spaceId}`, @@ -263,9 +255,12 @@ export const wrapSuppressedSequenceAlerts = ({ [ALERT_URL]: alertUrl, }, }; - } - ); + }); - return [...acc, ...wrappedBuildingBlocks, sequenceAlert]; - }, []); + return [...acc, ...wrappedBuildingBlocks, sequenceAlert] as Array< + WrappedFieldsLatest + >; + }, + [] as Array> + ); }; From b065fb68f3c4b1c3a0b73faf91c5a6b9b941b9d8 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 9 Sep 2024 15:35:46 -0400 Subject: [PATCH 11/70] fix bug where building block alerts were not being added to individual suppressed alert --- .../rule_types/utils/bulk_create_with_suppression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index a398ba5683341..1609f4d54c398 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -119,7 +119,7 @@ export const bulkCreateWithSuppression = async < // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); return ( - isEqlBuildingBlockAlert(newAlertSource) && + isEqlBuildingBlockAlert(someAlert?._source) && isEqlShellAlert(newAlertSource) && someAlert?._source?.[ALERT_GROUP_ID] === newAlertSource?.[ALERT_GROUP_ID] ); From 19334e45678cec7e3d6bcfc1bf0b18fd4977cffb Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 9 Sep 2024 15:44:57 -0400 Subject: [PATCH 12/70] working test with feature flag enabled --- .../config/ess/config.base.ts | 1 + .../configs/serverless.config.ts | 5 +- .../execution_logic/eql_alert_suppression.ts | 58 +++++-------------- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index 6cd2702857e0f..38d3277766e10 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -85,6 +85,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'riskScoringPersistence', 'riskScoringRoutesEnabled', 'manualRuleRunEnabled', + 'alertSuppressionForSequenceEqlRuleEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index 6691a781d4e1e..0f99b3a3eeebf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -17,6 +17,9 @@ export default createTestConfig({ 'testing_ignored.constant', '/testing_regex*/', ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['manualRuleRunEnabled'])}`, + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'manualRuleRunEnabled', + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index fc21a95afb799..20c0aae51aab0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -54,6 +54,8 @@ const getSequenceQuery = (id: string) => export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); + const esDeleteAllIndices = getService('esDeleteAllIndices'); + const es = getService('es'); const log = getService('log'); const { @@ -76,8 +78,12 @@ export default ({ getService }: FtrProviderContext) => { }); afterEach(async () => { - await deleteAllAlerts(supertest, log, es); + await deleteAllAlerts(supertest, log, es, [ + '.preview.alerts-security.alerts-*', + '.alerts-security.alerts-*', + ]); await deleteAllRules(supertest, log); + await esDeleteAllIndices('.preview.alerts*'); }); describe('non-sequence queries', () => { @@ -1788,7 +1794,7 @@ export default ({ getService }: FtrProviderContext) => { }); describe('sequence queries', () => { - it.skip('only suppresses alerts within the rule execution', async () => { + it('only suppresses alerts within the rule execution', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; const laterTimestamp = '2020-10-28T06:50:00.000Z'; @@ -1808,7 +1814,7 @@ export default ({ getService }: FtrProviderContext) => { doc1, doc1WithLaterTimestamp, { ...doc1, '@timestamp': timestamp1 }, - // { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, + { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, ]); const rule: EqlRuleCreateProps = { @@ -1833,8 +1839,8 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - // we expect three alerts, two building block and - // one sequence alert, let's confirm that + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, @@ -1842,7 +1848,6 @@ export default ({ getService }: FtrProviderContext) => { ); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); - console.error(JSON.stringify(sequenceAlert)); expect(sequenceAlert[0]?._source).toEqual({ ...sequenceAlert[0]?._source, @@ -1854,45 +1859,10 @@ export default ({ getService }: FtrProviderContext) => { ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: timestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp, // suppression ends with later timestamp - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - it('logs a warning if suppression is configured', async () => { - const id = uuidv4(); - await indexGeneratedSourceDocuments({ - docsCount: 10, - seed: () => ({ id }), + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['agent.name'], - duration: { - value: 300, - unit: 'm', - }, - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { logs } = await previewRule({ - supertest, - rule, - invocationCount: 1, - }); - - const [{ warnings }] = logs; - - expect(warnings).toContain( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - ); }); }); }); From 1a6d913fd8e7ea0470eea01ee41d202e93c49fec Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 17 Sep 2024 18:08:32 -0400 Subject: [PATCH 13/70] adds more ftr tests with additional bug fixes --- .../create_persistence_rule_type_wrapper.ts | 54 +- .../eql/build_alert_group_from_sequence.ts | 1 + ...bulk_create_suppressed_alerts_in_memory.ts | 22 +- .../utils/bulk_create_with_suppression.ts | 9 +- .../sequence_eql_query_no_duration.json | 2 +- .../execution_logic/eql.ts | 1 - .../execution_logic/eql_alert_suppression.ts | 1198 ++++++++++++++++- .../test/security_solution_cypress/config.ts | 3 + 8 files changed, 1259 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index c89c3f4fdcea7..0ea087f557a7d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -25,6 +25,8 @@ import { TIMESTAMP, VERSION, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_SUPPRESSION_VALUE, + ALERT_SUPPRESSION_TERMS, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; // import { @@ -182,6 +184,11 @@ export const suppressAlertsInMemory = < (alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; const suppressionDocsCount = alert._source[ALERT_SUPPRESSION_DOCS_COUNT]; + console.error('INSIDE SUPPRESSION DOCS COUNT', suppressionDocsCount); + console.error( + 'NOT A BUILDING BLOCK', + alert._source['kibana.alert.building_block_type'] == null + ); const suppressionEnd = alert._source[ALERT_SUPPRESSION_END]; if (instanceId && idsMap[instanceId] != null) { @@ -200,6 +207,8 @@ export const suppressAlertsInMemory = < [] ); + console.error('WHAT IS IDS MAP', JSON.stringify(idsMap)); + const alertCandidates = filteredAlerts.map((alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; if (instanceId) { @@ -209,6 +218,11 @@ export const suppressAlertsInMemory = < return alert; }); + console.error( + 'INSIDE SUPPRESSION DOCS COUNT', + alertCandidates.map((alrt) => alrt._source[ALERT_SUPPRESSION_DOCS_COUNT]) + ); + return { alertCandidates, suppressedAlerts, @@ -495,6 +509,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return acc; }, {}); + console.error('ARE THERE EXISTING ALERTS', existingAlertsByInstanceId.length); + // filter out alerts that were already suppressed // alert was suppressed if its suppression ends is older // than suppression end of existing alert @@ -603,12 +619,10 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); - // console.error( - // 'WHAT IS AUGMENTED ALERT', - // augmentedAlerts.map( - // (augAlert) => augAlert._source._source['kibana.alert.group.id'] - // ) - // ); + console.error( + 'WHAT IS AUGMENTED ALERT LENGTH', + augmentedAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) + ); /** * buildingBlockAlerts?.filter((someAlert) => { @@ -629,7 +643,9 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper // only create building block alerts for the new eql shell alert const matchingBuildingBlockAlerts = newAlerts?.length > 0 && getMatchingBuildingBlockAlerts != null - ? getMatchingBuildingBlockAlerts(newAlerts[0]?._source) + ? newAlerts.flatMap((newAlert) => + getMatchingBuildingBlockAlerts(newAlert?._source) + ) : []; const augmentedBuildingBlockAlerts = @@ -655,9 +671,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper body: [ ...duplicateAlertUpdates, ...mapAlertsToBulkCreate(augmentedAlerts), - ...(newAlerts?.length > 0 - ? mapAlertsToBulkCreate(augmentedBuildingBlockAlerts) - : []), + ...mapAlertsToBulkCreate(augmentedBuildingBlockAlerts), ], refresh: true, }); @@ -686,8 +700,24 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 ) // Security solution's EQL rule consists of building block alerts which should be filtered out. - // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. - .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); + .filter((alert) => alert['kibana.alert.building_block_type'] == null); + + console.error('HOW MANY CREATED ALERTS', createdAlerts.length); + console.error('HOW MANY DUPLICATE ALERTS', duplicateAlerts.length); + console.error( + 'HOW MANY SUPPRESSED IN MEMORY ALERTS', + suppressedInMemoryAlerts.length + ); + + console.error( + 'created alerts suppression count', + JSON.stringify( + createdAlerts.map((alrt) => ({ + 'suppressed values': alrt[ALERT_SUPPRESSION_TERMS], + 'suppressed count': alrt['kibana.alert.suppression.docs_count'], + })) + ) + ); createdAlerts.forEach((alert) => options.services.alertFactory diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 7bf6cb1b26113..73540553b07f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -78,6 +78,7 @@ export const buildAlertGroupFromSequence = ( ) ); } catch (error) { + console.error(`\n************\nBIG ERROR ${error}\n************\n`); ruleExecutionLogger.error(error); return []; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 227c5cf213b95..5b6ef3536b00f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -8,7 +8,11 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import { ALERT_BUILDING_BLOCK_TYPE, ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import { + ALERT_BUILDING_BLOCK_TYPE, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, +} from '@kbn/rule-data-utils'; import partition from 'lodash/partition'; import type { @@ -196,7 +200,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ if (!suppressOnMissingFields) { sequences.forEach((sequence) => { - // if none of the events in the sequence + // if none of the events in the given sequence // contain a value, then wrap sequence normally, // otherwise wrap as suppressed // ask product @@ -209,7 +213,9 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ if (eventsWithFields.length === 0) { // unsuppressible sequence alert - unsuppressibleWrappedDocs.concat(...wrapSequences([sequence], buildReasonMessage)); + const wrappedSequence = wrapSequences([sequence], buildReasonMessage); + // console.error('UNSUPPRESSED WRAP SEQUENCE', wrappedSequence); + unsuppressibleWrappedDocs.push(...wrappedSequence); console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); } else { suppressibleSequences.push(sequence); @@ -237,10 +243,15 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ ); console.error( - 'SUPPRESSIBLE WRAPPED original time', - sequenceAlerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) + 'SUPPRESSIBLE WRAPPED values', + sequenceAlerts.map((doc) => doc._source[ALERT_SUPPRESSION_TERMS]) ); + // console.error( + // 'SUPPRESSIBLE WRAPPED original time', + // sequenceAlerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) + // ); + // the code in executeBulkCreateAlerts should // not have to change, and might even allow me to remove // some of my earlier changes where the fields property was not available @@ -315,6 +326,7 @@ export const executeBulkCreateAlerts = async < : tuple.from.toISOString(); if (unsuppressibleWrappedDocs.length) { + console.error('UNSUPPRESSIBLE DOCS WRAPPED'); const unsuppressedResult = await bulkCreate( unsuppressibleWrappedDocs, tuple.maxSignals - toReturn.createdSignalsCount, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 1609f4d54c398..17cbd63a1b3fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -107,7 +107,14 @@ export const bulkCreateWithSuppression = async < console.error( 'DO WRAPPED DOCS HAVE INSTANCE IDS', - wrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) + wrappedDocs.reduce( + (acc, doc) => ({ + ...acc, + [doc._source[ALERT_INSTANCE_ID]]: + acc[doc._source[ALERT_INSTANCE_ID]] != null ? acc[doc._source[ALERT_INSTANCE_ID]] + 1 : 0, + }), + {} + ) ); let myfunc; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json index 32dc6a75d140b..03a66efe5dc4d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json @@ -2,7 +2,7 @@ "name": "EQL sequence rule", "description": "Rule with an eql query", "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], - "rule_id": "rule-id-eql-1", + "rule_id": "rule-id-eql-2", "enabled": true, "index": ["auditbeat*", "packetbeat*"], "interval": "2m", diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index 39a2c1874f7f9..6a59d5244b88c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -510,7 +510,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - // TODO: investigate original_time here too it('generates shell alerts from EQL sequences in the expected form', async () => { const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['auditbeat-*']), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index 20c0aae51aab0..99b7ebe93e45f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -50,6 +50,7 @@ import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/ut const getQuery = (id: string) => `any where id == "${id}"`; const getSequenceQuery = (id: string) => `sequence [any where id == "${id}"] [any where id == "${id}"]`; +const getSequenceQueryTrue = () => `sequence [any where true] [any where true]`; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); @@ -1793,12 +1794,11 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('sequence queries', () => { - it('only suppresses alerts within the rule execution', async () => { + describe.only('sequence queries "per rule execution" suppression duration', () => { + it('suppresses alerts in a given rule execution', async () => { const id = uuidv4(); - const timestamp = '2020-10-28T06:45:00.000Z'; - const laterTimestamp = '2020-10-28T06:50:00.000Z'; - const timestamp1 = '2020-10-28T06:51:00.000Z'; + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; const doc1 = { id, @@ -1810,12 +1810,94 @@ export default ({ getService }: FtrProviderContext) => { '@timestamp': laterTimestamp, }; - await indexListOfSourceDocuments([ - doc1, - doc1WithLaterTimestamp, - { ...doc1, '@timestamp': timestamp1 }, - { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, - ]); + const doc2WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp2, + }; + + // sequence alert 1 is made up of doc1 and doc1WithLaterTimestamp, + // sequence alert 2 is made up of doc1WithLaterTimestamp and doc2WithLaterTimestamp + // sequence alert 2 is suppressed because it shares the same + // host.name value as sequence alert 1 + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithLaterTimestamp]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts in a given rule execution when a subsequent event for an sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // the reason for the second alert is because despite the value being null + // in one of the two events in the sequence, the sequence alert will + // adopt the value for host.name and be suppressible. + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), @@ -1861,6 +1943,1100 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_SUPPRESSION_START]: laterTimestamp, [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts in a given rule execution when doNotSuppress is set and one event in the sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // the reason for the second alert is because despite the value being null + // in one of the two events in the sequence, the sequence alert will + // adopt the value for host.name and be suppressible. + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + }); + + it('suppresses alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(6); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); + + expect(sequenceAlert[1]?._source).toEqual({ + ...sequenceAlert[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp2, + [ALERT_SUPPRESSION_END]: laterTimestamp3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when "doNotSuppress" is set and suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: undefined }, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: { name: undefined }, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: { name: undefined }, + }; + + // first suppressible sequence alert will be doc1, doc1WithNoHost + // two unsuppressible sequence alerts will consist of + // [doc1WithNoHost, doc2WithNoHost] and [doc2WithNoHost, doc3WithNoHost] + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect two unsuppressed alerts and one suppressed alert + // for a total of 3 sequence alerts and two building block alerts per alert + // for a total of 6 building block alerts. Let's confirm that + expect(previewAlerts.length).toEqual(9); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlert.length).toEqual(3); + + expect(sequenceAlert[2]?._source).toEqual({ + ...sequenceAlert[2]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + }); + + it('does not suppress alerts outside of the current rule execution search range', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:05:00.000Z'; // this should not count towards events + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + const timestamp1 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + { ...doc1, '@timestamp': timestamp1 }, + { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp1, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts on a field with array values', async () => { + const id = uuidv4(); + + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:46:00.000Z'; + const timestamp3 = '2020-10-28T06:47:00.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: ['host-a', 'host-b'] }, + }; + + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': timestamp2 }, + { ...doc1, '@timestamp': timestamp3 }, + ]); + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + // query: `sequence [any where true] [any where true]`, + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(previewAlerts.length).toEqual(3); // one sequence, two building block + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a', 'host-b'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp2, + [ALERT_SUPPRESSION_END]: timestamp3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts with missing fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:47:00.000Z'; + const timestamp3 = '2020-10-28T06:48:00.000Z'; + const timestamp4 = '2020-10-28T06:49:00.000Z'; + const timestamp5 = '2020-10-28T06:50:00.000Z'; + const timestamp6 = '2020-10-28T06:51:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': timestamp4, + agent: { name: 'agent-1' }, + }; + + // 2 alert should be suppressed: 1 doc and 1 doc2 + // we have 5 sequences here + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': timestamp2 }, + { ...doc1, '@timestamp': timestamp3 }, + doc2, + { ...doc2, '@timestamp': timestamp5 }, + { ...doc2, '@timestamp': timestamp6 }, + ]); + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + console.error('HOW MANY SEQUENCE ALERTS', sequenceAlert.length); + console.error( + 'agent values for sequence alerts', + sequenceAlert.map((alrt) => JSON.stringify(alrt?._source?.[ALERT_SUPPRESSION_TERMS])) + ); + + console.error('HOW MANY buildingBlockAlerts', buildingBlockAlerts.length); + console.error( + 'HOW MANY buildingBlockAlerts', + buildingBlockAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) + ); + expect(previewAlerts.length).toEqual(6); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp2, + [ALERT_SUPPRESSION_END]: timestamp3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[1]._source).toEqual({ + ...sequenceAlert[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-1'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp4, + [ALERT_SUPPRESSION_END]: timestamp6, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + // continue testing here: + it('suppresses alerts with missing fields and multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; + + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + id: uuidv4(), + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + id: uuidv4(), + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + id: uuidv4(), + agent: undefined, + }; + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, + + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, + + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, + + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + // query: getSequenceQuery(id), + query: getSequenceQueryTrue(), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + console.error('HOW MANY SEQUENCE ALERTS', sequenceAlert.length); + console.error( + 'agent values for sequence alerts', + sequenceAlert.map((alrt) => JSON.stringify(alrt?._source?.[ALERT_SUPPRESSION_TERMS])) + ); + + console.error('HOW MANY buildingBlockAlerts', buildingBlockAlerts.length); + console.error( + 'HOW MANY buildingBlockAlerts', + buildingBlockAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) + ); + + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(4); + sequenceAlert.forEach((alrt) => + expect(alrt?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(1) + ); + expect(previewAlerts.length).toEqual(10); + expect(sequenceAlert[3]._source).toEqual({ + ...sequenceAlert[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[1]._source).toEqual({ + ...sequenceAlert[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[2]._source).toEqual({ + ...sequenceAlert[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[3]._source).toEqual({ + ...sequenceAlert[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts with missing fields if configured as such', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + agent: { name: 'agent-1' }, + }; + + // 1 alert should be suppressed: 1 doc only + await indexListOfSourceDocuments([doc1, doc1, doc1, doc2, doc2, doc2]); + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(3); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-1'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('does not suppress alerts with missing fields and multiple suppress by fields if configured as such', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + // 1 alert should be suppressed: 1 doc only + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + noMissingFieldsDoc, + noMissingFieldsDoc, + + missingNameFieldsDoc, + missingNameFieldsDoc, + missingNameFieldsDoc, + + missingVersionFieldsDoc, + missingVersionFieldsDoc, + missingVersionFieldsDoc, + + missingAgentFieldsDoc, + missingAgentFieldsDoc, + missingAgentFieldsDoc, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getQuery(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // from 7 injected, only one should be suppressed + expect(previewAlerts.length).toEqual(6); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('deduplicates multiple alerts while suppressing on rule interval only', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + // 4 alert should be suppressed + await indexListOfSourceDocuments([ + doc1, + doc1, + doc1, + doc1, + doc2, + doc2, + doc2, + doc2, + doc2, + doc2, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-50m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('deduplicates a single alert while suppressing new ones', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + // 3 alerts should be suppressed + await indexListOfSourceDocuments([doc1, doc1, doc2, doc2, doc2, doc2]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('creates and suppresses an alert into alert created on previous execution', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + const doc2 = { + ...doc1, + '@timestamp': secondTimestamp, + }; + + // 1 created + 1 suppressed on first run + // 1 created + 2 suppressed on second run + await indexListOfSourceDocuments([doc1, doc1, doc1, doc2, doc2, doc2, doc2]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); }); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 58d873369d99d..b403afd471e5e 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -57,6 +57,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.fleet.developer.bundledPackageLocation=./inexistentDir`, '--csp.strict=false', '--csp.warnLegacyBrowsers=false', + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, ], }, }; From 3bfa5ab4b5f30737a002db4ab8bf93885e835cd4 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Wed, 18 Sep 2024 22:52:29 -0400 Subject: [PATCH 14/70] adds more tests --- ...bulk_create_suppressed_alerts_in_memory.ts | 8 +- .../execution_logic/eql_alert_suppression.ts | 493 ++++-------------- 2 files changed, 119 insertions(+), 382 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 5b6ef3536b00f..8395a8b022359 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -242,10 +242,10 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null ); - console.error( - 'SUPPRESSIBLE WRAPPED values', - sequenceAlerts.map((doc) => doc._source[ALERT_SUPPRESSION_TERMS]) - ); + // console.error( + // 'SUPPRESSIBLE WRAPPED values', + // JSON.stringify(sequenceAlerts.map((doc) => doc._source[ALERT_SUPPRESSION_TERMS])) + // ); // console.error( // 'SUPPRESSIBLE WRAPPED original time', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index 99b7ebe93e45f..a6be5a4bd0206 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -2071,16 +2071,16 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); + expect(previewAlerts.length).toEqual(6); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, + expect(sequenceAlert[1]?._source).toEqual({ + ...sequenceAlert[1]?._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', @@ -2331,28 +2331,33 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('suppresses alerts on a field with array values', async () => { + it('does not suppress alerts where no suppression field values match', async () => { const id = uuidv4(); - - const timestamp = '2020-10-28T06:45:00.000Z'; - const timestamp2 = '2020-10-28T06:46:00.000Z'; - const timestamp3 = '2020-10-28T06:47:00.000Z'; - + const timestamp = '2020-10-28T06:50:00.000Z'; // this should not count towards events + const laterTimestamp = '2020-10-28T06:50:01.000Z'; + const timestamp1 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; const doc1 = { id, '@timestamp': timestamp, - host: { name: ['host-a', 'host-b'] }, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: 'host-b' }, }; await indexListOfSourceDocuments([ doc1, - { ...doc1, '@timestamp': timestamp2 }, - { ...doc1, '@timestamp': timestamp3 }, + doc1WithLaterTimestamp, + { ...doc1, '@timestamp': timestamp1, host: { name: 'host-c' } }, + { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2, host: { name: 'host-d' } }, ]); + const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), query: getSequenceQuery(id), - // query: `sequence [any where true] [any where true]`, alert_suppression: { group_by: ['host.name'], missing_fields_strategy: 'suppress', @@ -2372,62 +2377,44 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(9); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(previewAlerts.length).toEqual(3); // one sequence, two building block - expect(sequenceAlert[0]._source).toEqual({ - ...sequenceAlert[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a', 'host-b'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp2, - [ALERT_SUPPRESSION_END]: timestamp3, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlert.length).toEqual(3); + + expect(sequenceAlert[0]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(sequenceAlert[1]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(sequenceAlert[2]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); }); - it('suppresses alerts with missing fields', async () => { + it('suppresses alerts on a field with array values', async () => { const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; - const timestamp2 = '2020-10-28T06:47:00.000Z'; - const timestamp3 = '2020-10-28T06:48:00.000Z'; - const timestamp4 = '2020-10-28T06:49:00.000Z'; - const timestamp5 = '2020-10-28T06:50:00.000Z'; - const timestamp6 = '2020-10-28T06:51:00.000Z'; + const timestamp2 = '2020-10-28T06:46:00.000Z'; + const timestamp3 = '2020-10-28T06:47:00.000Z'; + const doc1 = { id, '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - - const doc2 = { - ...doc1, - '@timestamp': timestamp4, - agent: { name: 'agent-1' }, + host: { name: ['host-a', 'host-b'] }, }; - // 2 alert should be suppressed: 1 doc and 1 doc2 - // we have 5 sequences here await indexListOfSourceDocuments([ doc1, { ...doc1, '@timestamp': timestamp2 }, { ...doc1, '@timestamp': timestamp3 }, - doc2, - { ...doc2, '@timestamp': timestamp5 }, - { ...doc2, '@timestamp': timestamp6 }, ]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), query: getSequenceQuery(id), alert_suppression: { - group_by: ['agent.name'], + group_by: ['host.name'], missing_fields_strategy: 'suppress', }, from: 'now-35m', @@ -2443,30 +2430,19 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, - sort: ['agent.name', ALERT_ORIGINAL_TIME], + sort: [ALERT_ORIGINAL_TIME], }); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - console.error('HOW MANY SEQUENCE ALERTS', sequenceAlert.length); - console.error( - 'agent values for sequence alerts', - sequenceAlert.map((alrt) => JSON.stringify(alrt?._source?.[ALERT_SUPPRESSION_TERMS])) - ); - - console.error('HOW MANY buildingBlockAlerts', buildingBlockAlerts.length); - console.error( - 'HOW MANY buildingBlockAlerts', - buildingBlockAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) - ); - expect(previewAlerts.length).toEqual(6); + expect(previewAlerts.length).toEqual(3); // one sequence, two building block expect(sequenceAlert[0]._source).toEqual({ ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { - field: 'agent.name', - value: null, + field: 'host.name', + value: ['host-a', 'host-b'], }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', @@ -2475,24 +2451,8 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_END]: timestamp3, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); - - expect(sequenceAlert[1]._source).toEqual({ - ...sequenceAlert[1]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-1'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp4, - [ALERT_SUPPRESSION_END]: timestamp6, - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, - }); }); - // continue testing here: it('suppresses alerts with missing fields and multiple suppress by fields', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; @@ -2517,19 +2477,16 @@ export default ({ getService }: FtrProviderContext) => { const missingNameFieldsDoc = { ...noMissingFieldsDoc, - id: uuidv4(), agent: { version: 10 }, }; const missingVersionFieldsDoc = { ...noMissingFieldsDoc, - id: uuidv4(), agent: { name: 'agent-a' }, }; const missingAgentFieldsDoc = { ...noMissingFieldsDoc, - id: uuidv4(), agent: undefined, }; @@ -2554,8 +2511,7 @@ export default ({ getService }: FtrProviderContext) => { const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), - // query: getSequenceQuery(id), - query: getSequenceQueryTrue(), + query: getSequenceQuery(id), alert_suppression: { group_by: ['agent.name', 'agent.version'], missing_fields_strategy: 'suppress', @@ -2573,32 +2529,18 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, + size: 100, sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding }); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - console.error('HOW MANY SEQUENCE ALERTS', sequenceAlert.length); - console.error( - 'agent values for sequence alerts', - sequenceAlert.map((alrt) => JSON.stringify(alrt?._source?.[ALERT_SUPPRESSION_TERMS])) - ); - - console.error('HOW MANY buildingBlockAlerts', buildingBlockAlerts.length); - console.error( - 'HOW MANY buildingBlockAlerts', - buildingBlockAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) - ); // for sequence alerts if neither of the fields are there, we cannot suppress? expect(sequenceAlert.length).toEqual(4); - sequenceAlert.forEach((alrt) => - expect(alrt?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(1) - ); - expect(previewAlerts.length).toEqual(10); - expect(sequenceAlert[3]._source).toEqual({ - ...sequenceAlert[3]._source, + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', @@ -2609,7 +2551,7 @@ export default ({ getService }: FtrProviderContext) => { value: ['10'], }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, }); expect(sequenceAlert[1]._source).toEqual({ @@ -2617,11 +2559,11 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: ['agent-a'], + value: null, }, { field: 'agent.version', - value: null, + value: ['10'], }, ], [ALERT_SUPPRESSION_DOCS_COUNT]: 1, @@ -2632,14 +2574,14 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: null, + value: ['agent-a'], }, { field: 'agent.version', - value: ['10'], + value: null, }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); expect(sequenceAlert[3]._source).toEqual({ @@ -2658,75 +2600,21 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('does not suppress alerts with missing fields if configured as such', async () => { + it('does not suppress alerts when "doNotSuppress" is set and we have alerts with missing fields and multiple suppress by fields', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:45:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - - const doc2 = { - ...doc1, - agent: { name: 'agent-1' }, - }; - - // 1 alert should be suppressed: 1 doc only - await indexListOfSourceDocuments([doc1, doc1, doc1, doc2, doc2, doc2]); - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['agent.name'], - missing_fields_strategy: 'doNotSuppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: ['agent.name', ALERT_ORIGINAL_TIME], - }); - expect(previewAlerts.length).toEqual(3); - expect(previewAlerts[0]._source).toEqual({ - ...previewAlerts[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-1'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_ORIGINAL_TIME]: timestamp, - [ALERT_SUPPRESSION_START]: timestamp, - [ALERT_SUPPRESSION_END]: timestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - - // rest of alerts are not suppressed and do not have suppress properties - previewAlerts.slice(1).forEach((previewAlert) => { - const source = previewAlert._source; - expect(source).toHaveProperty('id', id); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); - }); - }); + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; - it('does not suppress alerts with missing fields and multiple suppress by fields if configured as such', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:45:00.000Z'; const noMissingFieldsDoc = { id, '@timestamp': timestamp, @@ -2749,31 +2637,30 @@ export default ({ getService }: FtrProviderContext) => { agent: undefined, }; - // 1 alert should be suppressed: 1 doc only + // 4 alerts should be suppressed: 1 for each pair of documents await indexListOfSourceDocuments([ noMissingFieldsDoc, - noMissingFieldsDoc, - noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, - missingNameFieldsDoc, - missingNameFieldsDoc, - missingNameFieldsDoc, + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, - missingVersionFieldsDoc, - missingVersionFieldsDoc, - missingVersionFieldsDoc, + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, - missingAgentFieldsDoc, - missingAgentFieldsDoc, - missingAgentFieldsDoc, + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, ]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getQuery(id), + query: getSequenceQuery(id), alert_suppression: { group_by: ['agent.name', 'agent.version'], - missing_fields_strategy: 'doNotSuppress', }, from: 'now-35m', @@ -2789,12 +2676,22 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, - sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + size: 100, }); - // from 7 injected, only one should be suppressed - expect(previewAlerts.length).toEqual(6); - expect(previewAlerts[0]._source).toEqual({ - ...previewAlerts[0]._source, + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(9); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + expect(suppressedSequenceAlerts[0]._source).toEqual({ + ...suppressedSequenceAlerts[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', @@ -2805,17 +2702,7 @@ export default ({ getService }: FtrProviderContext) => { value: ['10'], }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - - // rest of alerts are not suppressed and do not have suppress properties - previewAlerts.slice(1).forEach((previewAlert) => { - const source = previewAlert._source; - expect(source).toHaveProperty('id', id); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); - expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); }); @@ -2823,29 +2710,27 @@ export default ({ getService }: FtrProviderContext) => { const id = uuidv4(); const firstTimestamp = '2020-10-28T05:45:00.000Z'; const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const secondTimestamp2 = '2020-10-28T06:10:01.000Z'; + const secondTimestamp3 = '2020-10-28T06:10:02.000Z'; + const secondTimestamp4 = '2020-10-28T06:10:03.000Z'; + const secondTimestamp5 = '2020-10-28T06:10:04.000Z'; + const secondTimestamp6 = '2020-10-28T06:10:05.000Z'; + const doc1 = { id, '@timestamp': firstTimestamp, host: { name: 'host-a' }, }; - const doc2 = { - ...doc1, - '@timestamp': secondTimestamp, - }; - // 4 alert should be suppressed await indexListOfSourceDocuments([ doc1, - doc1, - doc1, - doc1, - doc2, - doc2, - doc2, - doc2, - doc2, - doc2, + { ...doc1, '@timestamp': secondTimestamp }, + { ...doc1, '@timestamp': secondTimestamp2 }, + { ...doc1, '@timestamp': secondTimestamp3 }, + { ...doc1, '@timestamp': secondTimestamp4 }, + { ...doc1, '@timestamp': secondTimestamp5 }, + { ...doc1, '@timestamp': secondTimestamp6 }, ]); const rule: EqlRuleCreateProps = { @@ -2864,180 +2749,32 @@ export default ({ getService }: FtrProviderContext) => { supertest, rule, timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), - invocationCount: 2, + invocationCount: 2, // 2 invocations so we can see that the same sequences are not generated / suppressed again. }); const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['host.name', ALERT_ORIGINAL_TIME], }); - expect(previewAlerts.length).toEqual(2); - expect(previewAlerts[0]._source).toEqual({ - ...previewAlerts[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: firstTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - - expect(previewAlerts[1]._source).toEqual({ - ...previewAlerts[1]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_ORIGINAL_TIME]: secondTimestamp, - [ALERT_SUPPRESSION_START]: secondTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, - }); - }); - - it('deduplicates a single alert while suppressing new ones', async () => { - const id = uuidv4(); - const firstTimestamp = '2020-10-28T05:45:00.000Z'; - const secondTimestamp = '2020-10-28T06:10:00.000Z'; - const doc1 = { - id, - '@timestamp': firstTimestamp, - host: { name: 'host-a' }, - }; - - const doc2 = { - ...doc1, - '@timestamp': secondTimestamp, - }; - - // 3 alerts should be suppressed - await indexListOfSourceDocuments([doc1, doc1, doc2, doc2, doc2, doc2]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - // large look-back time covers all docs - from: 'now-1h', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), - invocationCount: 2, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: ['host.name', ALERT_ORIGINAL_TIME], - }); - expect(previewAlerts.length).toEqual(2); - expect(previewAlerts[0]._source).toEqual({ - ...previewAlerts[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: firstTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, - }); - expect(previewAlerts[1]._source).toEqual({ - ...previewAlerts[1]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_ORIGINAL_TIME]: secondTimestamp, - [ALERT_SUPPRESSION_START]: secondTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, - }); - }); - - it('creates and suppresses an alert into alert created on previous execution', async () => { - const id = uuidv4(); - const firstTimestamp = '2020-10-28T05:45:00.000Z'; - const secondTimestamp = '2020-10-28T06:10:00.000Z'; - const doc1 = { - id, - '@timestamp': firstTimestamp, - host: { name: 'host-a' }, - }; - - const doc2 = { - ...doc1, - '@timestamp': secondTimestamp, - }; - - // 1 created + 1 suppressed on first run - // 1 created + 2 suppressed on second run - await indexListOfSourceDocuments([doc1, doc1, doc1, doc2, doc2, doc2, doc2]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), - invocationCount: 2, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: ['host.name', ALERT_ORIGINAL_TIME], - }); - expect(previewAlerts.length).toEqual(2); - expect(previewAlerts[0]._source).toEqual({ - ...previewAlerts[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_ORIGINAL_TIME]: firstTimestamp, - [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: firstTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - expect(previewAlerts[1]._source).toEqual({ - ...previewAlerts[1]._source, + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(1); + expect(previewAlerts.length).toEqual(3); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', value: ['host-a'], }, ], - [ALERT_ORIGINAL_TIME]: secondTimestamp, [ALERT_SUPPRESSION_START]: secondTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_SUPPRESSION_END]: secondTimestamp6, + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // if deduplication failed this would 12, or the previewAlerts count would be double }); }); }); From 6b74bb4013a7d4d90efeffd0f77577253c93de3d Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 24 Sep 2024 11:26:35 -0400 Subject: [PATCH 15/70] adds test for not suppressing outside of duration --- .../rules/queries/sequence_eql_query.json | 4 +- .../execution_logic/eql_alert_suppression.ts | 1266 ++++++++++++++++- 2 files changed, 1267 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json index 74b6193c24531..ab9af79de9f34 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -16,7 +16,7 @@ }, "risk_score": 99, "to": "now", - "from": "now-90s", + "from": "now-70s", "severity": "high", "type": "eql", "language": "eql", @@ -55,7 +55,7 @@ "references": ["http://www.example.com/some-article-about-attack"], "alert_suppression": { "group_by": ["agent.name"], - "duration": { "value": 5, "unit": "h" }, + "duration": { "value": 5, "unit": "m" }, "missing_fields_strategy": "suppress" }, "version": 1 diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index a6be5a4bd0206..a0ce137ab6d97 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -1794,7 +1794,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe.only('sequence queries "per rule execution" suppression duration', () => { + describe('sequence queries with suppression "per rule execution"', () => { it('suppresses alerts in a given rule execution', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; @@ -2778,5 +2778,1269 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe.only('sequence queries with suppression duration', () => { + it('suppresses alerts across two rule executions when the suppression duration exceeds the rule interval', async () => { + const id = uuidv4(); + const firstTimestamp = new Date(Date.now() - 1000).toISOString(); + const firstTimestamp2 = new Date().toISOString(); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstDocument, + { ...firstDocument, '@timestamp': firstTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(alerts.hits.hits.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + alerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; + + expect(sequenceAlert[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp2, + [ALERT_SUPPRESSION_END]: firstTimestamp2, + [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + // index an event that happened 1 second before the next event in the sequence + const secondTimestamp = new Date(Date.now() - 1000).toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfSourceDocuments([secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + + const [sequenceAlert2] = partition( + secondAlerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + expect(sequenceAlert2.length).toEqual(1); + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same + [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed + }); + // suppression end value should be greater than second document timestamp, but lesser than current time + const suppressionEnd = new Date( + sequenceAlert2[0]._source?.[ALERT_SUPPRESSION_END] as string + ).getTime(); + expect(suppressionEnd).toBeLessThan(new Date().getTime()); + expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); + }); + + it('does not suppress alerts outside of duration', async () => { + const id = uuidv4(); + // this timestamp is 1 minute in the past + const firstTimestamp = new Date(Date.now() - 5000).toISOString(); + const firstTimestamp2 = new Date(Date.now() - 5500).toISOString(); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstDocument, + { ...firstDocument, '@timestamp': firstTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 7, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-10s', + interval: '5s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(alerts.hits.hits.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + alerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + + const secondTimestamp = new Date(Date.now() - 1000).toISOString(); + const secondTimestamp2 = new Date(Date.now()).toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. + // the second rule run should generate a new alert + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + + await indexListOfSourceDocuments([ + secondDocument, + { ...secondDocument, '@timestamp': secondTimestamp2 }, + ]); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(Date.now() + 1000); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + + const [sequenceAlert2] = partition( + secondAlerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + expect(sequenceAlert2.length).toEqual(2); + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed + }); + expect(sequenceAlert2[1]._source).toEqual({ + ...sequenceAlert2[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_SUPPRESSION_START]: secondTimestamp2, + [ALERT_SUPPRESSION_END]: secondTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed + }); + }); + + it('suppresses alerts across three non consecutive rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date(Date.now() - 1000).toISOString(); + const firstTimestamp2 = new Date().toISOString(); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstDocument, + { ...firstDocument, '@timestamp': firstTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(alerts.hits.hits.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + alerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; + + expect(sequenceAlert[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp2, + [ALERT_SUPPRESSION_END]: firstTimestamp2, + [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + const secondTimestamp = new Date(Date.now() - 1000).toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfSourceDocuments([secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + + const [sequenceAlert2] = partition( + secondAlerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + expect(sequenceAlert2.length).toEqual(1); + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same + [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed + }); + // suppression end value should be greater than second document timestamp, but lesser than current time + const suppressionEnd = new Date( + sequenceAlert2[0]._source?.[ALERT_SUPPRESSION_END] as string + ).getTime(); + expect(suppressionEnd).toBeLessThan(new Date().getTime()); + expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); + }); + + it('suppresses alerts in a given rule execution when a subsequent event for an sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // the reason for the second alert is because despite the value being null + // in one of the two events in the sequence, the sequence alert will + // adopt the value for host.name and be suppressible. + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts in a given rule execution when doNotSuppress is set and one event in the sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // the reason for the second alert is because despite the value being null + // in one of the two events in the sequence, the sequence alert will + // adopt the value for host.name and be suppressible. + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(6); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); + + expect(sequenceAlert[1]?._source).toEqual({ + ...sequenceAlert[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + }); + + it('suppresses alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(6); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); + + expect(sequenceAlert[1]?._source).toEqual({ + ...sequenceAlert[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp2, + [ALERT_SUPPRESSION_END]: laterTimestamp3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when "doNotSuppress" is set and suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: undefined }, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: { name: undefined }, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: { name: undefined }, + }; + + // first suppressible sequence alert will be doc1, doc1WithNoHost + // two unsuppressible sequence alerts will consist of + // [doc1WithNoHost, doc2WithNoHost] and [doc2WithNoHost, doc3WithNoHost] + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect two unsuppressed alerts and one suppressed alert + // for a total of 3 sequence alerts and two building block alerts per alert + // for a total of 6 building block alerts. Let's confirm that + expect(previewAlerts.length).toEqual(9); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlert.length).toEqual(3); + + expect(sequenceAlert[2]?._source).toEqual({ + ...sequenceAlert[2]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: laterTimestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + }); + + it('does not suppress alerts outside of the current rule execution search range', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:05:00.000Z'; // this should not count towards events + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + const timestamp1 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + { ...doc1, '@timestamp': timestamp1 }, + { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp1, + [ALERT_SUPPRESSION_END]: laterTimestamp2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts where no suppression field values match', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; // this should not count towards events + const laterTimestamp = '2020-10-28T06:50:01.000Z'; + const timestamp1 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: 'host-b' }, + }; + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + { ...doc1, '@timestamp': timestamp1, host: { name: 'host-c' } }, + { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2, host: { name: 'host-d' } }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(9); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlert.length).toEqual(3); + + expect(sequenceAlert[0]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(sequenceAlert[1]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(sequenceAlert[2]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + }); + + it('suppresses alerts on a field with array values', async () => { + const id = uuidv4(); + + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:46:00.000Z'; + const timestamp3 = '2020-10-28T06:47:00.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: ['host-a', 'host-b'] }, + }; + + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': timestamp2 }, + { ...doc1, '@timestamp': timestamp3 }, + ]); + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(previewAlerts.length).toEqual(3); // one sequence, two building block + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a', 'host-b'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_START]: timestamp2, + [ALERT_SUPPRESSION_END]: timestamp3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts with missing fields and multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; + + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, + + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, + + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, + + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 100, + sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(4); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + + expect(sequenceAlert[1]._source).toEqual({ + ...sequenceAlert[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[2]._source).toEqual({ + ...sequenceAlert[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sequenceAlert[3]._source).toEqual({ + ...sequenceAlert[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when "doNotSuppress" is set and we have alerts with missing fields and multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; + + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, + + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, + + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, + + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 100, + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(9); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + expect(suppressedSequenceAlerts[0]._source).toEqual({ + ...suppressedSequenceAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: ['agent-a'], + }, + { + field: 'agent.version', + value: ['10'], + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('deduplicates multiple alerts while suppressing on rule interval only', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const secondTimestamp2 = '2020-10-28T06:10:01.000Z'; + const secondTimestamp3 = '2020-10-28T06:10:02.000Z'; + const secondTimestamp4 = '2020-10-28T06:10:03.000Z'; + const secondTimestamp5 = '2020-10-28T06:10:04.000Z'; + const secondTimestamp6 = '2020-10-28T06:10:05.000Z'; + + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + // 4 alert should be suppressed + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': secondTimestamp }, + { ...doc1, '@timestamp': secondTimestamp2 }, + { ...doc1, '@timestamp': secondTimestamp3 }, + { ...doc1, '@timestamp': secondTimestamp4 }, + { ...doc1, '@timestamp': secondTimestamp5 }, + { ...doc1, '@timestamp': secondTimestamp6 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-50m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, // 2 invocations so we can see that the same sequences are not generated / suppressed again. + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(sequenceAlert.length).toEqual(1); + expect(previewAlerts.length).toEqual(3); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp6, + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // if deduplication failed this would 12, or the previewAlerts count would be double + }); + }); + }); }); }; From 4f4273323ea66c6d76826a9dfd8ce1ae6db290fd Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 24 Sep 2024 16:10:19 -0400 Subject: [PATCH 16/70] passing tests --- .../execution_logic/eql_alert_suppression.ts | 1024 +---------------- 1 file changed, 1 insertion(+), 1023 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index a0ce137ab6d97..718e0eab1c6bc 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -2779,7 +2779,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe.only('sequence queries with suppression duration', () => { + describe('sequence queries with suppression duration', () => { it('suppresses alerts across two rule executions when the suppression duration exceeds the rule interval', async () => { const id = uuidv4(); const firstTimestamp = new Date(Date.now() - 1000).toISOString(); @@ -3019,1028 +3019,6 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed }); }); - - it('suppresses alerts across three non consecutive rule executions', async () => { - const id = uuidv4(); - const firstTimestamp = new Date(Date.now() - 1000).toISOString(); - const firstTimestamp2 = new Date().toISOString(); - - const firstDocument = { - id, - '@timestamp': firstTimestamp, - host: { - name: 'host-a', - }, - }; - await indexListOfSourceDocuments([ - firstDocument, - { ...firstDocument, '@timestamp': firstTimestamp2 }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - duration: { - value: 300, - unit: 'm', - }, - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - const createdRule = await createRule(supertest, log, rule); - const alerts = await getOpenAlerts(supertest, log, es, createdRule); - - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(alerts.hits.hits.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - alerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); - - // suppression start equal to alert timestamp - const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; - - expect(sequenceAlert[0]._source).toEqual( - expect.objectContaining({ - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - // suppression boundaries equal to original event time, since no alert been suppressed - [ALERT_SUPPRESSION_START]: firstTimestamp2, - [ALERT_SUPPRESSION_END]: firstTimestamp2, - [TIMESTAMP]: suppressionStart, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, - }) - ); - - const secondTimestamp = new Date(Date.now() - 1000).toISOString(); - const secondDocument = { - id, - '@timestamp': secondTimestamp, - host: { - name: 'host-a', - }, - }; - - // Add a new document, then disable and re-enable to trigger another rule run. The second doc should - // trigger an update to the existing alert without changing the timestamp - await indexListOfSourceDocuments([secondDocument]); - await patchRule(supertest, log, { id: createdRule.id, enabled: false }); - await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp = new Date(); - const secondAlerts = await getOpenAlerts( - supertest, - log, - es, - createdRule, - RuleExecutionStatusEnum.succeeded, - undefined, - afterTimestamp - ); - - const [sequenceAlert2] = partition( - secondAlerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - - expect(sequenceAlert2.length).toEqual(1); - expect(sequenceAlert2[0]._source).toEqual({ - ...sequenceAlert2[0]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same - [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed - }); - // suppression end value should be greater than second document timestamp, but lesser than current time - const suppressionEnd = new Date( - sequenceAlert2[0]._source?.[ALERT_SUPPRESSION_END] as string - ).getTime(); - expect(suppressionEnd).toBeLessThan(new Date().getTime()); - expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); - }); - - it('suppresses alerts in a given rule execution when a subsequent event for an sequence has the suppression field undefined', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; - const laterTimestamp = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithLaterTimestamp = { - ...doc1, - '@timestamp': laterTimestamp, - }; - - const doc2WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp2, - host: undefined, - }; - - // sequence alert 1 will be doc1 and doc1WithLaterTimestamp - // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost - // the reason for the second alert is because despite the value being null - // in one of the two events in the sequence, the sequence alert will - // adopt the value for host.name and be suppressible. - - await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); - - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp2, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('suppresses alerts in a given rule execution when doNotSuppress is set and one event in the sequence has the suppression field undefined', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; - const laterTimestamp = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithLaterTimestamp = { - ...doc1, - '@timestamp': laterTimestamp, - }; - - const doc2WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp2, - host: undefined, - }; - - // sequence alert 1 will be doc1 and doc1WithLaterTimestamp - // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost - // the reason for the second alert is because despite the value being null - // in one of the two events in the sequence, the sequence alert will - // adopt the value for host.name and be suppressible. - await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'doNotSuppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); - - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp2, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('does not suppress alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; - const laterTimestamp = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp, - host: undefined, - }; - - const doc2WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp2, - host: undefined, - }; - - await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'doNotSuppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(6); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(4); - expect(sequenceAlert.length).toEqual(2); - - expect(sequenceAlert[1]?._source).toEqual({ - ...sequenceAlert[1]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, - }); - }); - - it('suppresses alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; - const laterTimestamp = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; - - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp, - host: undefined, - }; - - const doc2WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp2, - host: undefined, - }; - - const doc3WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp3, - host: undefined, - }; - - await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(6); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(4); - expect(sequenceAlert.length).toEqual(2); - - expect(sequenceAlert[1]?._source).toEqual({ - ...sequenceAlert[1]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: null, - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp2, - [ALERT_SUPPRESSION_END]: laterTimestamp3, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('does not suppress alerts when "doNotSuppress" is set and suppression field value is undefined for a sequence alert in a given rule execution', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; - const laterTimestamp = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; - - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp, - host: { name: undefined }, - }; - - const doc2WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp2, - host: { name: undefined }, - }; - - const doc3WithNoHost = { - ...doc1, - '@timestamp': laterTimestamp3, - host: { name: undefined }, - }; - - // first suppressible sequence alert will be doc1, doc1WithNoHost - // two unsuppressible sequence alerts will consist of - // [doc1WithNoHost, doc2WithNoHost] and [doc2WithNoHost, doc3WithNoHost] - - await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'doNotSuppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect two unsuppressed alerts and one suppressed alert - // for a total of 3 sequence alerts and two building block alerts per alert - // for a total of 6 building block alerts. Let's confirm that - expect(previewAlerts.length).toEqual(9); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(6); - expect(sequenceAlert.length).toEqual(3); - - expect(sequenceAlert[2]?._source).toEqual({ - ...sequenceAlert[2]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, - }); - }); - - it('does not suppress alerts outside of the current rule execution search range', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:05:00.000Z'; // this should not count towards events - const laterTimestamp = '2020-10-28T06:50:00.000Z'; - const timestamp1 = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithLaterTimestamp = { - ...doc1, - '@timestamp': laterTimestamp, - }; - - await indexListOfSourceDocuments([ - doc1, - doc1WithLaterTimestamp, - { ...doc1, '@timestamp': timestamp1 }, - { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); - - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp1, - [ALERT_SUPPRESSION_END]: laterTimestamp2, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('does not suppress alerts where no suppression field values match', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; // this should not count towards events - const laterTimestamp = '2020-10-28T06:50:01.000Z'; - const timestamp1 = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - }; - const doc1WithLaterTimestamp = { - ...doc1, - '@timestamp': laterTimestamp, - host: { name: 'host-b' }, - }; - - await indexListOfSourceDocuments([ - doc1, - doc1WithLaterTimestamp, - { ...doc1, '@timestamp': timestamp1, host: { name: 'host-c' } }, - { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2, host: { name: 'host-d' } }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(9); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(buildingBlockAlerts.length).toEqual(6); - expect(sequenceAlert.length).toEqual(3); - - expect(sequenceAlert[0]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); - expect(sequenceAlert[1]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); - expect(sequenceAlert[2]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); - }); - - it('suppresses alerts on a field with array values', async () => { - const id = uuidv4(); - - const timestamp = '2020-10-28T06:45:00.000Z'; - const timestamp2 = '2020-10-28T06:46:00.000Z'; - const timestamp3 = '2020-10-28T06:47:00.000Z'; - - const doc1 = { - id, - '@timestamp': timestamp, - host: { name: ['host-a', 'host-b'] }, - }; - - await indexListOfSourceDocuments([ - doc1, - { ...doc1, '@timestamp': timestamp2 }, - { ...doc1, '@timestamp': timestamp3 }, - ]); - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: [ALERT_ORIGINAL_TIME], - }); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - expect(previewAlerts.length).toEqual(3); // one sequence, two building block - expect(sequenceAlert[0]._source).toEqual({ - ...sequenceAlert[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a', 'host-b'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp2, - [ALERT_SUPPRESSION_END]: timestamp3, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('suppresses alerts with missing fields and multiple suppress by fields', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:45:00.000Z'; - const timestamp2 = '2020-10-28T06:45:01.000Z'; - const timestamp3 = '2020-10-28T06:45:02.000Z'; - const timestamp4 = '2020-10-28T06:45:03.000Z'; - const timestamp5 = '2020-10-28T06:45:04.000Z'; - const timestamp6 = '2020-10-28T06:45:05.000Z'; - const timestamp7 = '2020-10-28T06:45:06.000Z'; - const timestamp8 = '2020-10-28T06:45:07.000Z'; - const timestamp9 = '2020-10-28T06:45:08.000Z'; - const timestamp10 = '2020-10-28T06:45:09.000Z'; - const timestamp11 = '2020-10-28T06:45:10.000Z'; - const timestamp12 = '2020-10-28T06:45:11.000Z'; - - const noMissingFieldsDoc = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - agent: { name: 'agent-a', version: 10 }, - }; - - const missingNameFieldsDoc = { - ...noMissingFieldsDoc, - agent: { version: 10 }, - }; - - const missingVersionFieldsDoc = { - ...noMissingFieldsDoc, - agent: { name: 'agent-a' }, - }; - - const missingAgentFieldsDoc = { - ...noMissingFieldsDoc, - agent: undefined, - }; - - // 4 alerts should be suppressed: 1 for each pair of documents - await indexListOfSourceDocuments([ - noMissingFieldsDoc, - { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, - { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, - - { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, - { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, - { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, - - { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, - { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, - { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, - - { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, - { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, - { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['agent.name', 'agent.version'], - missing_fields_strategy: 'suppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - size: 100, - sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding - }); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - - // for sequence alerts if neither of the fields are there, we cannot suppress? - expect(sequenceAlert.length).toEqual(4); - expect(sequenceAlert[0]._source).toEqual({ - ...sequenceAlert[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-a'], - }, - { - field: 'agent.version', - value: ['10'], - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 3, - }); - - expect(sequenceAlert[1]._source).toEqual({ - ...sequenceAlert[1]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: null, - }, - { - field: 'agent.version', - value: ['10'], - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - - expect(sequenceAlert[2]._source).toEqual({ - ...sequenceAlert[2]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-a'], - }, - { - field: 'agent.version', - value: null, - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, - }); - - expect(sequenceAlert[3]._source).toEqual({ - ...sequenceAlert[3]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: null, - }, - { - field: 'agent.version', - value: null, - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, - }); - }); - - it('does not suppress alerts when "doNotSuppress" is set and we have alerts with missing fields and multiple suppress by fields', async () => { - const id = uuidv4(); - const timestamp = '2020-10-28T06:45:00.000Z'; - const timestamp2 = '2020-10-28T06:45:01.000Z'; - const timestamp3 = '2020-10-28T06:45:02.000Z'; - const timestamp4 = '2020-10-28T06:45:03.000Z'; - const timestamp5 = '2020-10-28T06:45:04.000Z'; - const timestamp6 = '2020-10-28T06:45:05.000Z'; - const timestamp7 = '2020-10-28T06:45:06.000Z'; - const timestamp8 = '2020-10-28T06:45:07.000Z'; - const timestamp9 = '2020-10-28T06:45:08.000Z'; - const timestamp10 = '2020-10-28T06:45:09.000Z'; - const timestamp11 = '2020-10-28T06:45:10.000Z'; - const timestamp12 = '2020-10-28T06:45:11.000Z'; - - const noMissingFieldsDoc = { - id, - '@timestamp': timestamp, - host: { name: 'host-a' }, - agent: { name: 'agent-a', version: 10 }, - }; - - const missingNameFieldsDoc = { - ...noMissingFieldsDoc, - agent: { version: 10 }, - }; - - const missingVersionFieldsDoc = { - ...noMissingFieldsDoc, - agent: { name: 'agent-a' }, - }; - - const missingAgentFieldsDoc = { - ...noMissingFieldsDoc, - agent: undefined, - }; - - // 4 alerts should be suppressed: 1 for each pair of documents - await indexListOfSourceDocuments([ - noMissingFieldsDoc, - { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, - { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, - - { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, - { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, - { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, - - { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, - { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, - { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, - - { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, - { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, - { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['agent.name', 'agent.version'], - missing_fields_strategy: 'doNotSuppress', - }, - from: 'now-35m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), - invocationCount: 1, - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - size: 100, - }); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - - // for sequence alerts if neither of the fields are there, we cannot suppress? - expect(sequenceAlert.length).toEqual(9); - const [suppressedSequenceAlerts] = partition( - sequenceAlert, - (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] - ); - expect(suppressedSequenceAlerts.length).toEqual(1); - expect(suppressedSequenceAlerts[0]._source).toEqual({ - ...suppressedSequenceAlerts[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'agent.name', - value: ['agent-a'], - }, - { - field: 'agent.version', - value: ['10'], - }, - ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, - }); - }); - - it('deduplicates multiple alerts while suppressing on rule interval only', async () => { - const id = uuidv4(); - const firstTimestamp = '2020-10-28T05:45:00.000Z'; - const secondTimestamp = '2020-10-28T06:10:00.000Z'; - const secondTimestamp2 = '2020-10-28T06:10:01.000Z'; - const secondTimestamp3 = '2020-10-28T06:10:02.000Z'; - const secondTimestamp4 = '2020-10-28T06:10:03.000Z'; - const secondTimestamp5 = '2020-10-28T06:10:04.000Z'; - const secondTimestamp6 = '2020-10-28T06:10:05.000Z'; - - const doc1 = { - id, - '@timestamp': firstTimestamp, - host: { name: 'host-a' }, - }; - - // 4 alert should be suppressed - await indexListOfSourceDocuments([ - doc1, - { ...doc1, '@timestamp': secondTimestamp }, - { ...doc1, '@timestamp': secondTimestamp2 }, - { ...doc1, '@timestamp': secondTimestamp3 }, - { ...doc1, '@timestamp': secondTimestamp4 }, - { ...doc1, '@timestamp': secondTimestamp5 }, - { ...doc1, '@timestamp': secondTimestamp6 }, - ]); - - const rule: EqlRuleCreateProps = { - ...getEqlRuleForAlertTesting(['ecs_compliant']), - query: getSequenceQuery(id), - alert_suppression: { - group_by: ['host.name'], - missing_fields_strategy: 'suppress', - }, - // large look-back time covers all docs - from: 'now-50m', - interval: '30m', - }; - - const { previewId } = await previewRule({ - supertest, - rule, - timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), - invocationCount: 2, // 2 invocations so we can see that the same sequences are not generated / suppressed again. - }); - const previewAlerts = await getPreviewAlerts({ - es, - previewId, - sort: ['host.name', ALERT_ORIGINAL_TIME], - }); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); - - // for sequence alerts if neither of the fields are there, we cannot suppress? - expect(sequenceAlert.length).toEqual(1); - expect(previewAlerts.length).toEqual(3); - expect(sequenceAlert[0]._source).toEqual({ - ...sequenceAlert[0]._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [ALERT_SUPPRESSION_START]: secondTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestamp6, - [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // if deduplication failed this would 12, or the previewAlerts count would be double - }); - }); }); }); }; From 88424c2feea28e4666efa21fc037370fcc606e90 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Wed, 25 Sep 2024 11:59:13 -0400 Subject: [PATCH 17/70] fixes lint and type check errors --- .../common/schemas/8.16.0/index.ts | 27 +++++++++ .../rule_registry/common/schemas/index.ts | 7 ++- .../create_persistence_rule_type_wrapper.ts | 58 ++---------------- .../server/utils/persistence_types.ts | 2 +- .../components/step_define_rule/index.tsx | 6 -- .../components/step_define_rule/schema.tsx | 1 - ...e_experimental_feature_fields_transform.ts | 1 - .../pages/rule_editing/index.tsx | 6 +- .../logic/use_alert_suppression.tsx | 1 - .../eql/build_alert_group_from_sequence.ts | 3 - .../detection_engine/rule_types/eql/eql.ts | 59 +++++-------------- .../rule_types/factories/utils/build_alert.ts | 12 ---- .../factories/utils/build_bulk_body.ts | 1 - .../group_and_bulk_create.ts | 2 - ...bulk_create_suppressed_alerts_in_memory.ts | 16 +---- .../utils/bulk_create_with_suppression.ts | 23 ++------ .../utils/get_is_alert_suppression_active.ts | 5 -- .../rule_types/utils/suppression_utils.ts | 6 -- .../utils/wrap_suppressed_alerts.ts | 11 +--- 19 files changed, 64 insertions(+), 183 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts diff --git a/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts new file mode 100644 index 0000000000000..9c2f2891f6700 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_BUILDING_BLOCK_TYPE } from '@kbn/rule-data-utils'; +import { AlertWithCommonFields880 } from '../8.8.0'; + +import { SuppressionFields8130 } from '../8.13.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.16.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.16.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields8160 extends SuppressionFields8130 { + [ALERT_BUILDING_BLOCK_TYPE]: undefined | string; +} + +export type AlertWithSuppressionFields8160 = AlertWithCommonFields880 & SuppressionFields8160; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index 5c168a4b899cc..0a76d43b2c9de 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -13,11 +13,12 @@ import type { CommonAlertFields880, } from './8.8.0'; -import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; +// import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; +import type { AlertWithSuppressionFields8160, SuppressionFields8160 } from './8.16.0'; export type { - AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest, - SuppressionFields8130 as SuppressionFieldsLatest, + AlertWithSuppressionFields8160 as AlertWithSuppressionFieldsLatest, + SuppressionFields8160 as SuppressionFieldsLatest, CommonAlertFieldName880 as CommonAlertFieldNameLatest, CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest, CommonAlertFields880 as CommonAlertFieldsLatest, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 0ea087f557a7d..9b5458adc7585 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -25,8 +25,6 @@ import { TIMESTAMP, VERSION, ALERT_RULE_EXECUTION_TIMESTAMP, - ALERT_SUPPRESSION_VALUE, - ALERT_SUPPRESSION_TERMS, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; // import { @@ -184,11 +182,11 @@ export const suppressAlertsInMemory = < (alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; const suppressionDocsCount = alert._source[ALERT_SUPPRESSION_DOCS_COUNT]; - console.error('INSIDE SUPPRESSION DOCS COUNT', suppressionDocsCount); - console.error( - 'NOT A BUILDING BLOCK', - alert._source['kibana.alert.building_block_type'] == null - ); + // console.error('INSIDE SUPPRESSION DOCS COUNT', suppressionDocsCount); + // console.error( + // 'NOT A BUILDING BLOCK', + // alert._source['kibana.alert.building_block_type'] == null + // ); const suppressionEnd = alert._source[ALERT_SUPPRESSION_END]; if (instanceId && idsMap[instanceId] != null) { @@ -207,7 +205,7 @@ export const suppressAlertsInMemory = < [] ); - console.error('WHAT IS IDS MAP', JSON.stringify(idsMap)); + // console.error('WHAT IS IDS MAP', JSON.stringify(idsMap)); const alertCandidates = filteredAlerts.map((alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; @@ -218,11 +216,6 @@ export const suppressAlertsInMemory = < return alert; }); - console.error( - 'INSIDE SUPPRESSION DOCS COUNT', - alertCandidates.map((alrt) => alrt._source[ALERT_SUPPRESSION_DOCS_COUNT]) - ); - return { alertCandidates, suppressedAlerts, @@ -240,8 +233,6 @@ export const isExistingDateGtEqThanAlert = < property: typeof ALERT_SUPPRESSION_END | typeof ALERT_SUPPRESSION_START ) => { const existingDate = existingAlert?._source?.[property]; - console.error('WHAT IS THE EXISTING DATE', existingDate); - console.error('WHAT IS THE SOURCE PROPERTY', alert._source[property]); return existingDate ? existingDate >= alert._source[property].toISOString() : false; }; @@ -509,8 +500,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper return acc; }, {}); - console.error('ARE THERE EXISTING ALERTS', existingAlertsByInstanceId.length); - // filter out alerts that were already suppressed // alert was suppressed if its suppression ends is older // than suppression end of existing alert @@ -558,8 +547,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper } }); - console.error('ARE THERE DUPLICATE ALERTS', duplicateAlerts?.length); - const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => { const existingAlert = existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; @@ -593,7 +580,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }); let enrichedAlerts = newAlerts; - console.error('WHAT ARE ENRICHED ALERTS LENGTH BEFORE', enrichedAlerts?.length); if (enrichAlerts) { try { @@ -605,8 +591,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper } } - console.error('WHAT ARE ENRICHED ALERTS LENGTH AFTER', enrichedAlerts?.length); - if (maxAlerts && enrichedAlerts.length > maxAlerts) { enrichedAlerts.length = maxAlerts; alertsWereTruncated = true; @@ -619,11 +603,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); - console.error( - 'WHAT IS AUGMENTED ALERT LENGTH', - augmentedAlerts.map((alrt) => alrt?._source?.[ALERT_SUPPRESSION_TERMS]) - ); - /** * buildingBlockAlerts?.filter((someAlert) => { // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); @@ -637,8 +616,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }); */ - console.error('is it defined', getMatchingBuildingBlockAlerts); - // TODO: move all this stuff into a utility function // only create building block alerts for the new eql shell alert const matchingBuildingBlockAlerts = @@ -661,12 +638,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }) : []; - // console.error('DO WE HAVE NEW ALERTS?', newAlerts?.length); - // console.error('DO WE HAVE BUILDING BLOCK ALERTS?? ', buildingBlockAlerts?.length); - // console.error( - // 'DO THE MAP ALERTS TO BULK CREATE CONTAIN THE INSTANCE IDS', - // JSON.stringify(mapAlertsToBulkCreate(augmentedAlerts)) - // ); const bulkResponse = await ruleDataClientWriter.bulk({ body: [ ...duplicateAlertUpdates, @@ -702,23 +673,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper // Security solution's EQL rule consists of building block alerts which should be filtered out. .filter((alert) => alert['kibana.alert.building_block_type'] == null); - console.error('HOW MANY CREATED ALERTS', createdAlerts.length); - console.error('HOW MANY DUPLICATE ALERTS', duplicateAlerts.length); - console.error( - 'HOW MANY SUPPRESSED IN MEMORY ALERTS', - suppressedInMemoryAlerts.length - ); - - console.error( - 'created alerts suppression count', - JSON.stringify( - createdAlerts.map((alrt) => ({ - 'suppressed values': alrt[ALERT_SUPPRESSION_TERMS], - 'suppressed count': alrt['kibana.alert.suppression.docs_count'], - })) - ) - ); - createdAlerts.forEach((alert) => options.services.alertFactory .create(alert._id) diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index b23efde7095fe..caf1df5551128 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -54,7 +54,7 @@ export type SuppressedAlertService = ( currentTimeOverride?: Date, isRuleExecutionOnly?: boolean, maxAlerts?: number, - getMatchingBuildingBlockAlerts?: (alert: unknown) => Array + getMatchingBuildingBlockAlerts?: (alert: unknown) => unknown[] ) => Promise>; export interface SuppressedAlertServiceResult extends PersistenceAlertServiceResult { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 79e262fd8ec1e..b8c59c5bd60a1 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -492,11 +492,6 @@ const StepDefineRuleComponent: FC = ({ isEqlSequenceQuery(queryBar?.query?.query as string) ); - useEffect( - () => console.error('IS SUPPRESSION ENABLED', isAlertSuppressionEnabled), - [isAlertSuppressionEnabled] - ); - /** If we don't have ML field information, users can't meaningfully interact with suppression fields */ const areSuppressionFieldsDisabledByMlFields = isMlRule(ruleType) && (mlRuleConfigLoading || !mlSuppressionFields.length); @@ -1198,7 +1193,6 @@ const StepDefineRuleReadOnlyComponent: FC = ({ const dataForDescription: Partial = getStepDataDataSource(data); const transformFields = useExperimentalFeatureFieldsTransform(); const fieldsToDisplay = transformFields(dataForDescription); - console.error('WHAT FIELDS TO DISPLAY', fieldsToDisplay.queryBar?.query.query); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index efc08a9b6c839..77faa57d51de3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -655,7 +655,6 @@ export const schema: FormSchema = { ): ReturnType> | undefined => { const [{ formData }] = args; const needsValidation = isSuppressionRuleConfiguredWithGroupBy(formData.ruleType); - console.error('DO WE NEED VALIDATION', needsValidation); if (!needsValidation) { return; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index 67221eb865569..a95106d5da817 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -21,7 +21,6 @@ export const useExperimentalFeatureFieldsTransform = { - console.error('fields query bar', fields.queryBar?.query?.query); const isSuppressionDisabled = isEqlRule(fields.ruleType) && isEqlSequenceQuery(fields.queryBar?.query?.query as string) && diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index a29146d99aa8b..e0d6ab6fa8076 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -55,10 +55,8 @@ import { MaxWidthEuiFlexItem, } from '../../../../detections/pages/detection_engine/rules/helpers'; import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations'; -import { - DefineStepRule, - RuleStep, -} from '../../../../detections/pages/detection_engine/rules/types'; +import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { RuleStep } from '../../../../detections/pages/detection_engine/rules/types'; import * as i18n from './translations'; import { SecurityPageName } from '../../../../app/types'; import { ruleStepsOrder } from '../../../../detections/pages/detection_engine/rules/utils'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index f712e6749f50e..e6159494b75fe 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -27,7 +27,6 @@ export const useAlertSuppression = ( } if (isEqlRule(ruleType) && isEqlSequenceQuery) { - console.error('IS SEQUENCE SUPPRESSION ENABLED', isAlertSuppressionForSequenceEQLRuleEnabled); return isAlertSuppressionForSequenceEQLRuleEnabled; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 73540553b07f0..4e91af23dc91b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -60,7 +60,6 @@ export const buildAlertGroupFromSequence = ( // since that's when the group ID is determined. let baseAlerts: BaseFieldsLatest[] = []; try { - console.error('CALLING WITHIN BUILD ALERT GROUP FROM SEQUENCE'); baseAlerts = sequence.events.map((event) => buildBulkBody( spaceId, @@ -78,7 +77,6 @@ export const buildAlertGroupFromSequence = ( ) ); } catch (error) { - console.error(`\n************\nBIG ERROR ${error}\n************\n`); ruleExecutionLogger.error(error); return []; } @@ -157,7 +155,6 @@ export const buildAlertRoot = ( severity: completeRule.ruleParams.severity, mergedDoc: mergedAlerts as SignalSourceHit, }); - console.error('CALLING INSIDE BUILD ALERT ROOT'); const doc = buildAlert( wrappedBuildingBlocks, completeRule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 7a23beb2ffe07..471d53a970382 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -162,54 +162,25 @@ export const eqlExecutor = async ({ newSignals = wrapHits(events, buildReasonMessageForEqlAlert); } } else if (sequences) { - console.error('isAlertSuppressionActive??? ', isAlertSuppressionActive); if ( isAlertSuppressionActive && experimentalFeatures.alertSuppressionForSequenceEqlRuleEnabled ) { - // result.warningMessages.push( - // 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - // ); - - console.error('how many sequences?', sequences.length); - - /* - We are missing the 'fields' property from the sequences.events because - we are wrapping the sequences before passing them to the bulk create - suppressed function. My hypothesis is to pass the raw sequence data to - bulk create suppressed alerts, then do the sequence wrapping - within the create suppressed alerts function. - - */ - - // commenting out all this code because it needs to happen further down the stack, - // where we are no longer relying on fields. - // const candidateSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); - // // partition sequence alert from building block alerts - // const [sequenceAlerts, buildingBlockAlerts] = partition( - // candidateSignals, - // (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null - // ); - // console.error('how many potential sequence alerts?', sequenceAlerts.length); - try { - await bulkCreateSuppressedSequencesInMemory({ - sequences, - toReturn: result, - wrapSequences, // TODO: fix type mismatch - bulkCreate, - services, - buildReasonMessage: buildReasonMessageForEqlAlert, - ruleExecutionLogger, - tuple, - alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedSequences, - alertTimestampOverride, - alertWithSuppression, - experimentalFeatures, - }); - } catch (exc) { - console.error('WHAT IS THE EXC', exc); - } + await bulkCreateSuppressedSequencesInMemory({ + sequences, + toReturn: result, + wrapSequences, // TODO: fix type mismatch + bulkCreate, + services, + buildReasonMessage: buildReasonMessageForEqlAlert, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedSequences, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + }); } else { newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index f0b633bb0225f..b0735b0391daf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -192,21 +192,11 @@ export const buildAlert = ( const params = completeRule.ruleParams; - console.error( - 'WHAT ARE DOCS ORIGINAL TIMES before', - docs.map((doc) => doc._source[TIMESTAMP]) - ); - const originalTime = getValidDateFromDoc({ doc: docs[0], primaryTimestamp: TIMESTAMP, }); - console.error( - 'WHAT ARE DOCS ORIGINAL TIMES after', - docs.map((doc) => doc._source[TIMESTAMP]) - ); - const timestamp = alertTimestampOverride?.toISOString() ?? new Date().toISOString(); const alertUrl = getAlertDetailsUrl({ @@ -217,8 +207,6 @@ export const buildAlert = ( spaceId, }); - console.error('WHAT IS ORIGINAL TIME', originalTime?.toISOString()); - return { [TIMESTAMP]: timestamp, [SPACE_IDS]: spaceId != null ? [spaceId] : [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts index 93efdc3639b29..9294cc7159c12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts @@ -106,7 +106,6 @@ export const buildBulkBody = ( const thresholdResult = mergedDoc._source?.threshold_result; if (isSourceDoc(mergedDoc)) { - console.error('CALLING INSIDE BUILD BULK BODY'); return { ...validatedSource, ...validatedEventFields, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts index 0870fc9a15b80..38018060e2252 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/group_and_bulk_create.ts @@ -251,8 +251,6 @@ export const groupAndBulkCreate = async ({ terms: Object.entries(bucket.key).map(([key, value]) => ({ field: key, value })), })); - console.error('SUPPRESSION BUCKETS', JSON.stringify(suppressionBuckets, null, 2)); - const wrappedAlerts = wrapSuppressedAlerts({ suppressionBuckets, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 8395a8b022359..925c8a89a7466 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -8,11 +8,7 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import { - ALERT_BUILDING_BLOCK_TYPE, - ALERT_INSTANCE_ID, - ALERT_SUPPRESSION_TERMS, -} from '@kbn/rule-data-utils'; +import { ALERT_BUILDING_BLOCK_TYPE } from '@kbn/rule-data-utils'; import partition from 'lodash/partition'; import type { @@ -38,7 +34,6 @@ import type { BaseFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; -import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -137,18 +132,12 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage); - console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); suppressibleEvents = partitionedEvents[0]; } // refactor the below into a separate function const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage); - console.error( - 'SUPPRESSIBLE WRAPPED instance ids', - suppressibleWrappedDocs.map((doc) => doc._source[ALERT_INSTANCE_ID]) - ); - // I think we create a separate bulkCreateSuppressedInMemory function // specifically for eql sequences // since sequences act differently from the other alerts. @@ -214,9 +203,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ if (eventsWithFields.length === 0) { // unsuppressible sequence alert const wrappedSequence = wrapSequences([sequence], buildReasonMessage); - // console.error('UNSUPPRESSED WRAP SEQUENCE', wrappedSequence); unsuppressibleWrappedDocs.push(...wrappedSequence); - console.error('unsuppressible docs', unsuppressibleWrappedDocs.length); } else { suppressibleSequences.push(sequence); } @@ -326,7 +313,6 @@ export const executeBulkCreateAlerts = async < : tuple.from.toISOString(); if (unsuppressibleWrappedDocs.length) { - console.error('UNSUPPRESSIBLE DOCS WRAPPED'); const unsuppressedResult = await bulkCreate( unsuppressibleWrappedDocs, tuple.maxSignals - toReturn.createdSignalsCount, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 17cbd63a1b3fa..d8dd4b7ff391a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -13,8 +13,12 @@ import type { AlertWithCommonFieldsLatest, SuppressionFieldsLatest, } from '@kbn/rule-registry-plugin/common/schemas'; -import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; +import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; +import { + isEqlBuildingBlockAlert, + isEqlShellAlert, +} from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { isQueryRule } from '../../../../../common/detection_engine/utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { makeFloatString } from './utils'; @@ -26,11 +30,6 @@ import type { RuleServices } from '../types'; import { createEnrichEventsFunction } from './enrichments'; import type { ExperimentalFeatures } from '../../../../../common'; import { getNumberOfSuppressedAlerts } from './get_number_of_suppressed_alerts'; -import { - isEqlBuildingBlockAlert, - isEqlShellAlert, -} from '@kbn/security-solution-plugin/common/api/detection_engine/model/alerts/8.0.0'; -import { ALERT_GROUP_ID } from '@kbn/security-solution-plugin/common/field_maps/field_names'; export interface GenericBulkCreateResponse { success: boolean; @@ -105,18 +104,6 @@ export const bulkCreateWithSuppression = async < } }; - console.error( - 'DO WRAPPED DOCS HAVE INSTANCE IDS', - wrappedDocs.reduce( - (acc, doc) => ({ - ...acc, - [doc._source[ALERT_INSTANCE_ID]]: - acc[doc._source[ALERT_INSTANCE_ID]] != null ? acc[doc._source[ALERT_INSTANCE_ID]] + 1 : 0, - }), - {} - ) - ); - let myfunc; if (buildingBlockAlerts != null && buildingBlockAlerts.length > 0) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts index 64783f86e5d91..5ab06db3043af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts @@ -28,15 +28,12 @@ export const getIsAlertSuppressionActive = async ({ alertSuppression, isFeatureDisabled = false, }: GetIsAlertSuppressionActiveParams) => { - console.error('IS IT DISABLED', isFeatureDisabled); if (isFeatureDisabled) { return false; } const isAlertSuppressionConfigured = Boolean(alertSuppression?.groupBy?.length); - console.error('IS IT CONFIGURED', isAlertSuppressionConfigured); - if (!isAlertSuppressionConfigured) { return false; } @@ -44,7 +41,5 @@ export const getIsAlertSuppressionActive = async ({ const license = await firstValueFrom(licensing.license$); const hasPlatinumLicense = license.hasAtLeast('platinum'); - console.error('hasPlatinumLicense?? ', hasPlatinumLicense); - return hasPlatinumLicense; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index fbb3ff2c853e1..03adf7ef8ff98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -41,10 +41,6 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp: string; instanceId: string; }) => { - console.error( - `primary timestamp: ${primaryTimestamp}, secondaryTimestamp: ${secondaryTimestamp}, fallbackTimestamp: ${fallbackTimestamp}` - ); - console.error('WHAT ARE FIELDS', JSON.stringify(fields)); const primeTimestamp = get(fields, primaryTimestamp) as string; const secondTimestamp = secondaryTimestamp != null ? (get(fields, secondaryTimestamp) as string) : fallbackTimestamp; @@ -61,8 +57,6 @@ export const getSuppressionAlertFields = ({ [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }; - console.error('WHAT ARE THE SUPPRESSION FIELDS', JSON.stringify(suppressionFields)); - return suppressionFields; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 51fd242febbfb..89b157e7c23b7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -31,12 +31,9 @@ import { generateBuildingBlockIds } from '../factories/utils/generate_building_b import type { BuildReasonMessage } from './reason_formatters'; import { buildAlertRoot } from '../eql/build_alert_group_from_sequence'; -import { getAlertDetailsUrl } from '@kbn/security-solution-plugin/common/utils/alert_detail_path'; -import { DEFAULT_ALERTS_INDEX } from '@kbn/security-solution-plugin/common/constants'; -import { - ALERT_GROUP_ID, - ALERT_GROUP_INDEX, -} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_path'; +import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants'; +import { ALERT_GROUP_ID, ALERT_GROUP_INDEX } from '../../../../../common/field_maps/field_names'; import { buildAncestors } from '../factories/utils/build_alert'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; @@ -168,7 +165,6 @@ export const wrapSuppressedSequenceAlerts = ({ return []; } - console.error('CALLING WITHIN WRAP SUPPRESSED SEQUENCE ALERTS'); // The "building block" alerts start out as regular BaseFields. // We'll add the group ID and index fields // after creating the shell alert later on @@ -191,7 +187,6 @@ export const wrapSuppressedSequenceAlerts = ({ ); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - console.error('sequence alert INSTANCE ID', instanceId); // The ID of each building block alert depends on all of the other building blocks as well, // so we generate the IDs after making all the BaseFields From 79917be176a65d3ceb09d47c7ea3f66d581636cb Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 26 Sep 2024 12:23:10 -0400 Subject: [PATCH 18/70] removes unused translations --- x-pack/plugins/translations/translations/fr-FR.json | 2 -- x-pack/plugins/translations/translations/ja-JP.json | 2 -- x-pack/plugins/translations/translations/zh-CN.json | 2 -- 3 files changed, 6 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index acf2acb2dbd47..f70044c7736a2 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -36211,8 +36211,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.enableThresholdSuppressionLabel": "Supprimer les alertes (version d'évaluation technique)", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "Requête EQL", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "Une requête EQL est requise.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "La suppression n'est pas prise en charge pour les requêtes de séquence EQL.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Remplacez la requête EQL par une requête non séquentielle ou supprimez les champs de suppression.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "Ouvrir une fenêtre contextuelle d'aide", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "Consultez {createEsqlRuleTypeLink} pour commencer à utiliser les règles ES|QL.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "documentation", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9e721bdec506a..bc80fdfd5522f 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -35954,8 +35954,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.enableThresholdSuppressionLabel": "アラートを抑制（テクニカルプレビュー）", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL クエリ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQLクエリは必須です。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQLシーケンスクエリでは抑制はサポートされていません。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} EQLクエリを非シーケンスクエリに変更するか、抑制フィールドを削除してください。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "ヘルプポップオーバーを開く", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "ES|QL ルールの使用を開始するには、{createEsqlRuleTypeLink}を確認してください。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "ドキュメンテーション", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9c6982f75c883..616136ebff152 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -35999,8 +35999,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.enableThresholdSuppressionLabel": "阻止告警（技术预览）", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL 查询", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQL 查询必填。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQL 序列查询不支持阻止。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} 将 EQL 查询更改为非序列查询，或移除阻止字段。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "打开帮助弹出框", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "请访问我们的{createEsqlRuleTypeLink}以开始使用 ES|QL 规则。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "文档", From 11c8a4b309e20eb4945b70e86a06702b59dccbfa Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 26 Sep 2024 14:13:22 -0400 Subject: [PATCH 19/70] missing intendedTimestamp from merge with main --- .../server/utils/create_persistence_rule_type_wrapper.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 4549fba7c0d3e..48655da690780 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -656,6 +656,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, + intendedTimestamp, }) : []; From 989988775355d1bfb8ac4bbc5dc4e60371f8653a Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 26 Sep 2024 16:19:52 -0400 Subject: [PATCH 20/70] fix type errors --- .../detection_engine/rule_types/eql/eql.test.ts | 1 + .../rule_types/utils/wrap_suppressed_alerts.ts | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index 6788deb97f65c..a571ca678948b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -183,6 +183,7 @@ describe('eql_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [], wrapSuppressedHits: jest.fn(), + wrapSuppressedSequences: jest.fn(), alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 9e9010a3c212b..085c8d5295aba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -171,20 +171,21 @@ export const wrapSuppressedSequenceAlerts = ({ // after creating the shell alert later on // since that's when the group ID is determined. const baseAlerts = sequence.events.map((event) => - buildBulkBody( + transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - 'placeholder-alert-uuid', // This is overriden below - publicBaseUrl - ) + alertUuid: 'placeholder-alert-uuid', // This is overriden below, + publicBaseUrl, + }) ); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); From 15e12ff1e82e70f9bbbdbeb8c54bbe2888c8cba6 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 26 Sep 2024 17:47:37 -0400 Subject: [PATCH 21/70] fix tests --- .../logic/use_alert_suppression.test.tsx | 4 ++ .../rule_types/eql/eql.test.ts | 51 ------------------- .../eql_rule_suppression_sequence.cy.ts | 24 +++++---- 3 files changed, 17 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index 949c957bf83c1..27018257179fb 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -6,9 +6,13 @@ */ import { renderHook } from '@testing-library/react-hooks'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import * as useIsExperimentalFeatureEnabledMock from '../../../common/hooks/use_experimental_features'; import { useAlertSuppression } from './use_alert_suppression'; describe('useAlertSuppression', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockReturnValue(false); ( [ 'new_terms', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index a571ca678948b..cdcd0145fd1ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -83,57 +83,6 @@ describe('eql_executor', () => { }`, ]); }); - - it('warns when a sequence query is used with alert suppression', async () => { - // mock a sequences response - alertServices.scopedClusterClient.asCurrentUser.eql.search.mockReset().mockResolvedValue({ - hits: { - total: { relation: 'eq', value: 10 }, - sequences: [], - }, - }); - - const ruleWithSequenceAndSuppression = getCompleteRuleMock({ - ...params, - query: 'sequence [any where true] [any where true]', - alertSuppression: { - groupBy: ['event.type'], - duration: { - value: 10, - unit: 'm', - }, - missingFieldsStrategy: 'suppress', - }, - }); - - const { result } = await eqlExecutor({ - inputIndex: DEFAULT_INDEX_PATTERN, - runtimeMappings: {}, - completeRule: ruleWithSequenceAndSuppression, - tuple, - ruleExecutionLogger, - services: alertServices, - version, - bulkCreate: jest.fn(), - wrapHits: jest.fn(), - wrapSequences: jest.fn(), - wrapSuppressedSequences: jest.fn(), - primaryTimestamp: '@timestamp', - exceptionFilter: undefined, - unprocessedExceptions: [], - wrapSuppressedHits: jest.fn(), - alertTimestampOverride: undefined, - alertWithSuppression: jest.fn(), - isAlertSuppressionActive: true, - experimentalFeatures: mockExperimentalFeatures, - scheduleNotificationResponseActionsService: - mockScheduleNotificationResponseActionsService, - }); - - expect(result.warningMessages).toContain( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - ); - }); }); it('should classify EQL verification exceptions as "user errors" when reporting to the framework', async () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts index 8f07781b7732c..b945f76b38310 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts @@ -12,16 +12,21 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { fillDefineEqlRule, selectEqlRuleType } from '../../../../tasks/create_new_rule'; -import { TOOLTIP } from '../../../../screens/common'; -import { - ALERT_SUPPRESSION_FIELDS, - ALERT_SUPPRESSION_FIELDS_INPUT, -} from '../../../../screens/create_new_rule'; +import { ALERT_SUPPRESSION_FIELDS_INPUT } from '../../../../screens/create_new_rule'; describe( 'Detection Rule Creation - EQL Rules - With Alert Suppression', { - tags: ['@ess', '@serverless'], + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, + ], + }, }, () => { describe('with sequence queries ', () => { @@ -36,11 +41,8 @@ describe( fillDefineEqlRule(rule); }); - it('disables the suppression fields and presents an informative tooltip', () => { - cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.disabled'); - - cy.get(ALERT_SUPPRESSION_FIELDS).trigger('mouseover'); - cy.get(TOOLTIP).contains('Suppression is not supported for EQL sequence queries.'); + it('displays the suppression fields', () => { + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); }); }); } From 8a38f2f84582b3ecefd0e7201bd7826b9b21914e Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 27 Sep 2024 13:40:14 -0400 Subject: [PATCH 22/70] test fixes --- .../rule_types/utils/suppression_utils.ts | 12 ++++++------ .../execution_logic/eql_alert_suppression.ts | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 03adf7ef8ff98..4540f9f6bb0d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -34,18 +34,18 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp, instanceId, }: { - fields: Record | undefined; + fields: Record | undefined; primaryTimestamp: string; secondaryTimestamp?: string; suppressionTerms: SuppressionTerm[]; fallbackTimestamp: string; instanceId: string; }) => { - const primeTimestamp = get(fields, primaryTimestamp) as string; - const secondTimestamp = - secondaryTimestamp != null ? (get(fields, secondaryTimestamp) as string) : fallbackTimestamp; - - const suppressionTime = new Date(primeTimestamp ?? secondTimestamp); + const suppressionTime = new Date( + get(fields, primaryTimestamp) ?? + (secondaryTimestamp && get(fields, secondaryTimestamp)) ?? + fallbackTimestamp + ); // console.error('SUPPRESSION TIME'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index 718e0eab1c6bc..c77701539238c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -2895,6 +2895,7 @@ export default ({ getService }: FtrProviderContext) => { expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); }); + // @skipInServerless it('does not suppress alerts outside of duration', async () => { const id = uuidv4(); // this timestamp is 1 minute in the past From d4701b4dcaaf007edf55c769be14dab658770f41 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 27 Sep 2024 15:08:06 -0400 Subject: [PATCH 23/70] fix type error --- .../create_persistence_rule_type_wrapper.ts | 23 +++++++------------ .../utils/wrap_suppressed_alerts.ts | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 0b891d50f1f7d..976ae77cc1288 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -104,14 +104,6 @@ const augmentAlerts = async ({ }; const mapAlertsToBulkCreate = (alerts: Array<{ _id: string; _source: T }>) => { - // const sourceNoId = (alertSource) => { - // if (Object.hasOwn(alertSource, '_id')) { - // const { _id, _index, _source, ...source } = alertSource; - // return { ..._source, ...source }; - // } - // return alertSource; - // }; - // return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, sourceNoId(alert._source)]); //sourceNoId(alert._source)]); return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]); }; @@ -644,18 +636,19 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper // only create building block alerts for the new eql shell alert const matchingBuildingBlockAlerts = newAlerts?.length > 0 && getMatchingBuildingBlockAlerts != null - ? newAlerts.flatMap((newAlert) => - getMatchingBuildingBlockAlerts(newAlert?._source) + ? newAlerts.flatMap( + (newAlert) => + getMatchingBuildingBlockAlerts(newAlert?._source) as Array<{ + _id: string; + _source: Record; + }> ) : []; const augmentedBuildingBlockAlerts = getMatchingBuildingBlockAlerts != null && newAlerts?.length > 0 - ? augmentAlerts({ - alerts: matchingBuildingBlockAlerts as unknown as Array<{ - _id: string; - _source: unknown; - }>, + ? await augmentAlerts({ + alerts: matchingBuildingBlockAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 085c8d5295aba..9b14223125509 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -155,7 +155,7 @@ export const wrapSuppressedSequenceAlerts = ({ (acc: Array>, sequence) => { const fields = sequence.events?.reduce( (seqAcc, event) => ({ ...seqAcc, ...event.fields }), - {} as Record + {} ); const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, From 50f1b24f1fd9dab18b4c837b65f3a30e63f8a731 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 08:11:50 -0400 Subject: [PATCH 24/70] skip test in serverless and fix conflicts and cypress config file --- .../execution_logic/eql_alert_suppression.ts | 5 ++--- x-pack/test/security_solution_cypress/config.ts | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index c77701539238c..aee49a76f5b25 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -1794,7 +1794,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('sequence queries with suppression "per rule execution"', () => { + describe('@skipInServerless sequence queries with suppression "per rule execution"', () => { it('suppresses alerts in a given rule execution', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; @@ -2779,7 +2779,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('sequence queries with suppression duration', () => { + describe('@skipInServerless sequence queries with suppression duration', () => { it('suppresses alerts across two rule executions when the suppression duration exceeds the rule interval', async () => { const id = uuidv4(); const firstTimestamp = new Date(Date.now() - 1000).toISOString(); @@ -2895,7 +2895,6 @@ export default ({ getService }: FtrProviderContext) => { expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); }); - // @skipInServerless it('does not suppress alerts outside of duration', async () => { const id = uuidv4(); // this timestamp is 1 minute in the past diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 1bed12e7710ed..d0c09f9c23263 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -47,6 +47,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'manualRuleRunEnabled', 'loggingRequestsEnabled', + 'alertSuppressionForSequenceEqlRuleEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests '--xpack.cloud.id=test', @@ -60,9 +61,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `--xpack.fleet.developer.bundledPackageLocation=./inexistentDir`, '--csp.strict=false', '--csp.warnLegacyBrowsers=false', - `--xpack.securitySolution.enableExperimental=${JSON.stringify([ - 'alertSuppressionForSequenceEqlRuleEnabled', - ])}`, ], }, }; From 992c983bb3b1a85c0b89f6d87d3787555fbe28dd Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 11:17:13 -0400 Subject: [PATCH 25/70] fix type errors in ftr test --- .../execution_logic/eql_alert_suppression.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index aee49a76f5b25..badb72f62e6ca 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -50,7 +50,6 @@ import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/ut const getQuery = (id: string) => `any where id == "${id}"`; const getSequenceQuery = (id: string) => `sequence [any where id == "${id}"] [any where id == "${id}"]`; -const getSequenceQueryTrue = () => `sequence [any where true] [any where true]`; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); @@ -2432,7 +2431,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - const [sequenceAlert, buildingBlockAlerts] = partition( + const [sequenceAlert] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); @@ -2532,7 +2531,7 @@ export default ({ getService }: FtrProviderContext) => { size: 100, sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding }); - const [sequenceAlert, buildingBlockAlerts] = partition( + const [sequenceAlert] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); @@ -2678,7 +2677,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, size: 100, }); - const [sequenceAlert, buildingBlockAlerts] = partition( + const [sequenceAlert] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); @@ -2756,7 +2755,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: ['host.name', ALERT_ORIGINAL_TIME], }); - const [sequenceAlert, buildingBlockAlerts] = partition( + const [sequenceAlert] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); From e34b4164fb76d85ba8c66a7043c9a12c7ddce422 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 14:49:46 -0400 Subject: [PATCH 26/70] more cleanup, move utility functions to utils file --- .../create_persistence_rule_type_wrapper.ts | 43 +------------------ .../model/alerts/8.0.0/index.ts | 15 ++----- .../detection_engine/rule_types/eql/eql.ts | 2 +- ...bulk_create_suppressed_alerts_in_memory.ts | 28 ------------ .../utils/bulk_create_with_suppression.ts | 6 +-- .../rule_types/utils/suppression_utils.ts | 2 - .../rule_types/utils/utils.ts | 21 ++++++++- 7 files changed, 26 insertions(+), 91 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 976ae77cc1288..f5d4c0a299fec 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -28,19 +28,12 @@ import { ALERT_INTENDED_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; -// import { -// ALERT_GROUP_ID, -// ALERT_ORIGINAL_TIME, -// } from '@kbn/security-solution-plugin/common/field_maps/field_names'; + import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; import { errorAggregator } from './utils'; import { AlertWithSuppressionFields870 } from '../../common/schemas/8.7.0'; -// import { -// isEqlBuildingBlockAlert, -// isEqlShellAlert, -// } from '@kbn/security-solution-plugin/common/api/detection_engine/model/alerts/8.0.0'; /** * Alerts returned from BE have date type coerced to ISO strings @@ -72,10 +65,6 @@ const augmentAlerts = async ({ currentTimeOverride: Date | undefined; intendedTimestamp: Date | undefined; }) => { - // console.error( - // 'DO AUGMENTED ALERTS HAVE INSTANCE IDS', - // alerts.map((alert) => alert._source[ALERT_INSTANCE_ID]) - // ); const commonRuleFields = getCommonAlertFields(options); const maintenanceWindowIds: string[] = alerts.length > 0 ? await options.services.getMaintenanceWindowIds() : []; @@ -185,11 +174,6 @@ export const suppressAlertsInMemory = < (alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; const suppressionDocsCount = alert._source[ALERT_SUPPRESSION_DOCS_COUNT]; - // console.error('INSIDE SUPPRESSION DOCS COUNT', suppressionDocsCount); - // console.error( - // 'NOT A BUILDING BLOCK', - // alert._source['kibana.alert.building_block_type'] == null - // ); const suppressionEnd = alert._source[ALERT_SUPPRESSION_END]; if (instanceId && idsMap[instanceId] != null) { @@ -208,8 +192,6 @@ export const suppressAlertsInMemory = < [] ); - // console.error('WHAT IS IDS MAP', JSON.stringify(idsMap)); - const alertCandidates = filteredAlerts.map((alert) => { const instanceId = alert._source[ALERT_INSTANCE_ID]; if (instanceId) { @@ -404,14 +386,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper maxAlerts, getMatchingBuildingBlockAlerts ) => { - // console.error( - // 'ALERT WITH SUPPRESSION INSTANCE IDS', - // alerts.map((doc) => doc._source[ALERT_INSTANCE_ID]) - // ); - // console.error( - // 'ALERT WITH SUPPRESSION original times', - // alerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) - // ); const ruleDataClientWriter = await ruleDataClient.getWriter({ namespace: options.spaceId, }); @@ -619,21 +593,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper intendedTimestamp, }); - /** - * buildingBlockAlerts?.filter((someAlert) => { - // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); - // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); - - return ( - isEqlBuildingBlockAlert(someAlert?._source) && - isEqlShellAlert(newAlerts?.[0]?._source) && - someAlert?._source?.[ALERT_GROUP_ID] === newAlerts[0]?._source[ALERT_GROUP_ID] - ); - }); - */ - - // TODO: move all this stuff into a utility function - // only create building block alerts for the new eql shell alert const matchingBuildingBlockAlerts = newAlerts?.length > 0 && getMatchingBuildingBlockAlerts != null ? newAlerts.flatMap( diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts index 24ed99b9386c9..6f9777c320734 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts @@ -37,10 +37,10 @@ import type { EVENT_KIND, SPACE_IDS, TIMESTAMP, + ALERT_BUILDING_BLOCK_TYPE, + ALERT_UUID, } from '@kbn/rule-data-utils'; -import { ALERT_BUILDING_BLOCK_TYPE, ALERT_UUID } from '@kbn/rule-data-utils'; - // TODO: Create and import 8.0.0 versioned ListArray schema import type { ListArray } from '@kbn/securitysolution-io-ts-list-types'; // TODO: Create and import 8.0.0 versioned alerting-types schemas @@ -68,8 +68,8 @@ import type { ALERT_RULE_TIMELINE_ID, ALERT_RULE_TIMELINE_TITLE, ALERT_RULE_TIMESTAMP_OVERRIDE, + ALERT_GROUP_ID, } from '../../../../../field_maps/field_names'; -import { ALERT_GROUP_ID } from '../../../../../field_maps/field_names'; // TODO: Create and import 8.0.0 versioned RuleAlertAction type import type { SearchTypes } from '../../../../../detection_engine/types'; import type { RuleAction } from '../../rule_schema'; @@ -164,17 +164,8 @@ export interface EqlShellFields800 extends BaseFields800 { export type EqlBuildingBlockAlert800 = AlertWithCommonFields800; -export const isEqlBuildingBlockAlert = ( - alertObject: unknown -): alertObject is EqlBuildingBlockAlert800 => - (alertObject as EqlBuildingBlockAlert800)?.[ALERT_BUILDING_BLOCK_TYPE] != null; - export type EqlShellAlert800 = AlertWithCommonFields800; -export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAlert800 => - (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && - (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; - export type GenericAlert800 = AlertWithCommonFields800; // This is the type of the final generated alert including base fields, common fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 8c98a1251d60b..7e2b0a5d44dac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -194,7 +194,7 @@ export const eqlExecutor = async ({ await bulkCreateSuppressedSequencesInMemory({ sequences, toReturn: result, - wrapSequences, // TODO: fix type mismatch + wrapSequences, bulkCreate, services, buildReasonMessage: buildReasonMessageForEqlAlert, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 925c8a89a7466..89a601c254eb8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -118,11 +118,6 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ let suppressibleEvents = enrichedEvents; let unsuppressibleWrappedDocs: Array> = []; - // if (suppressibleEvents?.[0]?.events != null) { - // // then we know that we have received sequences - // // so we must augment the wrapHits - // } - if (!suppressOnMissingFields) { const partitionedEvents = partitionMissingFieldsEvents( enrichedEvents, @@ -212,40 +207,17 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ suppressibleSequences = sequences; } - // refactor the below into a separate function const suppressibleWrappedDocs = wrapSuppressedSequences( suppressibleSequences, buildReasonMessage ); - // once we have wrapped thing similarly to - // build alert group from sequence, - // we can pass down the suppressibleWrappeDocs (our sequence alerts) - // and the building block alerts - // partition sequence alert from building block alerts const [sequenceAlerts, buildingBlockAlerts] = partition( suppressibleWrappedDocs, (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null ); - // console.error( - // 'SUPPRESSIBLE WRAPPED values', - // JSON.stringify(sequenceAlerts.map((doc) => doc._source[ALERT_SUPPRESSION_TERMS])) - // ); - - // console.error( - // 'SUPPRESSIBLE WRAPPED original time', - // sequenceAlerts.map((doc) => doc._source[ALERT_ORIGINAL_TIME]) - // ); - - // the code in executeBulkCreateAlerts should - // not have to change, and might even allow me to remove - // some of my earlier changes where the fields property was not available - - // I think we create a separate bulkCreateSuppressedInMemory function - // specifically for eql sequences - // since sequences act differently from the other alerts. return executeBulkCreateAlerts({ suppressibleWrappedDocs: sequenceAlerts, unsuppressibleWrappedDocs, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index d8dd4b7ff391a..d6ab64d985699 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -15,13 +15,9 @@ import type { } from '@kbn/rule-registry-plugin/common/schemas'; import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; -import { - isEqlBuildingBlockAlert, - isEqlShellAlert, -} from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { isQueryRule } from '../../../../../common/detection_engine/utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { makeFloatString } from './utils'; +import { makeFloatString, isEqlBuildingBlockAlert, isEqlShellAlert } from './utils'; import type { BaseFieldsLatest, WrappedFieldsLatest, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 4540f9f6bb0d3..175a629c2985c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -47,8 +47,6 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp ); - // console.error('SUPPRESSION TIME'); - const suppressionFields = { [ALERT_INSTANCE_ID]: instanceId, [ALERT_SUPPRESSION_TERMS]: suppressionTerms, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index 927a5170d293c..0414c4b34f901 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -11,7 +11,12 @@ import moment from 'moment'; import dateMath from '@kbn/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportResult } from '@elastic/elasticsearch'; -import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; +import { + ALERT_UUID, + ALERT_RULE_UUID, + ALERT_RULE_PARAMETERS, + ALERT_BUILDING_BLOCK_TYPE, +} from '@kbn/rule-data-utils'; import type { ListArray, ExceptionListItemSchema, @@ -67,6 +72,11 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants'; import type { GenericBulkCreateResponse } from '../factories'; +import type { + EqlBuildingBlockAlert800, + EqlShellAlert800, +} from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; +import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; export const MAX_RULE_GAP_RATIO = 4; @@ -1024,3 +1034,12 @@ export const getDisabledActionsWarningText = ({ return `${alertsGeneratedText} connector ${actionTypesJoined} is not enabled. To send notifications, you need a higher Security Analytics license / tier`; } }; + +export const isEqlBuildingBlockAlert = ( + alertObject: unknown +): alertObject is EqlBuildingBlockAlert800 => + (alertObject as EqlBuildingBlockAlert800)?.[ALERT_BUILDING_BLOCK_TYPE] != null; + +export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAlert800 => + (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && + (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; From 3bcbad5f2968120f12c388c278141edf617153b5 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 19:26:33 -0400 Subject: [PATCH 27/70] undo addition of building block type to alert schema --- .../common/schemas/8.16.0/index.ts | 27 ------------------- .../rule_registry/common/schemas/index.ts | 7 +++-- .../create_persistence_rule_type_wrapper.ts | 2 +- 3 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts diff --git a/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts deleted file mode 100644 index 9c2f2891f6700..0000000000000 --- a/x-pack/plugins/rule_registry/common/schemas/8.16.0/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ALERT_BUILDING_BLOCK_TYPE } from '@kbn/rule-data-utils'; -import { AlertWithCommonFields880 } from '../8.8.0'; - -import { SuppressionFields8130 } from '../8.13.0'; - -/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.16.0. -Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.16.0. - -If you are adding new fields for a new release of Kibana, create a new sibling folder to this one -for the version to be released and add the field(s) to the schema in that folder. - -Then, update `../index.ts` to import from the new folder that has the latest schemas, add the -new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. -*/ - -export interface SuppressionFields8160 extends SuppressionFields8130 { - [ALERT_BUILDING_BLOCK_TYPE]: undefined | string; -} - -export type AlertWithSuppressionFields8160 = AlertWithCommonFields880 & SuppressionFields8160; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index 0a76d43b2c9de..5c168a4b899cc 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -13,12 +13,11 @@ import type { CommonAlertFields880, } from './8.8.0'; -// import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; -import type { AlertWithSuppressionFields8160, SuppressionFields8160 } from './8.16.0'; +import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; export type { - AlertWithSuppressionFields8160 as AlertWithSuppressionFieldsLatest, - SuppressionFields8160 as SuppressionFieldsLatest, + AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest, + SuppressionFields8130 as SuppressionFieldsLatest, CommonAlertFieldName880 as CommonAlertFieldNameLatest, CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest, CommonAlertFields880 as CommonAlertFieldsLatest, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index f5d4c0a299fec..8117b0b111566 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -648,7 +648,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 ) // Security solution's EQL rule consists of building block alerts which should be filtered out. - .filter((alert) => alert['kibana.alert.building_block_type'] == null); + .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); createdAlerts.forEach((alert) => options.services.alertFactory From 63b09c4b2cbaf8d74c993fde171a3a8406ab79bf Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 20:04:08 -0400 Subject: [PATCH 28/70] more cleanup --- .../utils/create_persistence_rule_type_wrapper.ts | 4 +--- .../api/detection_engine/model/alerts/8.0.0/index.ts | 7 +++---- .../utils/bulk_create_suppressed_alerts_in_memory.ts | 7 ------- .../rule_types/utils/bulk_create_with_suppression.ts | 10 +++------- .../rule_types/utils/wrap_suppressed_alerts.ts | 1 - 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 8117b0b111566..c40331379aa0b 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -552,9 +552,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, { doc: { - // see if this is where the original time - // is not being set correctly - // when suppressing on per rule exec. ...getUpdatedSuppressionBoundaries( existingAlert, alert, @@ -648,6 +645,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper bulkResponse.body.items[idx + duplicateAlerts.length].create?.status === 201 ) // Security solution's EQL rule consists of building block alerts which should be filtered out. + // Building block alerts have additional "kibana.alert.group.index" attribute which is absent for the root alert. .filter((alert) => !Object.keys(alert).includes(ALERT_GROUP_INDEX)); createdAlerts.forEach((alert) => diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts index 6f9777c320734..3b7237ae8bb71 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts/8.0.0/index.ts @@ -6,6 +6,7 @@ */ import type { + ALERT_BUILDING_BLOCK_TYPE, ALERT_REASON, ALERT_RISK_SCORE, ALERT_RULE_AUTHOR, @@ -33,14 +34,12 @@ import type { ALERT_RULE_VERSION, ALERT_SEVERITY, ALERT_STATUS, + ALERT_UUID, ALERT_WORKFLOW_STATUS, EVENT_KIND, SPACE_IDS, TIMESTAMP, - ALERT_BUILDING_BLOCK_TYPE, - ALERT_UUID, } from '@kbn/rule-data-utils'; - // TODO: Create and import 8.0.0 versioned ListArray schema import type { ListArray } from '@kbn/securitysolution-io-ts-list-types'; // TODO: Create and import 8.0.0 versioned alerting-types schemas @@ -58,6 +57,7 @@ import type { ALERT_RULE_ACTIONS, ALERT_RULE_EXCEPTIONS_LIST, ALERT_RULE_FALSE_POSITIVES, + ALERT_GROUP_ID, ALERT_GROUP_INDEX, ALERT_RULE_IMMUTABLE, ALERT_RULE_MAX_SIGNALS, @@ -68,7 +68,6 @@ import type { ALERT_RULE_TIMELINE_ID, ALERT_RULE_TIMELINE_TITLE, ALERT_RULE_TIMESTAMP_OVERRIDE, - ALERT_GROUP_ID, } from '../../../../../field_maps/field_names'; // TODO: Create and import 8.0.0 versioned RuleAlertAction type import type { SearchTypes } from '../../../../../detection_engine/types'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 89a601c254eb8..c4c79d9c620a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -62,7 +62,6 @@ export interface BulkCreateSuppressedAlertsParams experimentalFeatures: ExperimentalFeatures; mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; - skipWrapping?: boolean; } export interface BulkCreateSuppressedSequencesParams @@ -85,7 +84,6 @@ export interface BulkCreateSuppressedSequencesParams experimentalFeatures: ExperimentalFeatures; mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; - skipWrapping?: boolean; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -95,7 +93,6 @@ export interface BulkCreateSuppressedSequencesParams export const bulkCreateSuppressedAlertsInMemory = async ({ enrichedEvents, buildingBlockAlerts, - skipWrapping = false, toReturn, wrapHits, bulkCreate, @@ -130,12 +127,8 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ suppressibleEvents = partitionedEvents[0]; } - // refactor the below into a separate function const suppressibleWrappedDocs = wrapSuppressedHits(suppressibleEvents, buildReasonMessage); - // I think we create a separate bulkCreateSuppressedInMemory function - // specifically for eql sequences - // since sequences act differently from the other alerts. return executeBulkCreateAlerts({ suppressibleWrappedDocs, unsuppressibleWrappedDocs, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index d6ab64d985699..5337e4c6ce376 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -100,14 +100,11 @@ export const bulkCreateWithSuppression = async < } }; - let myfunc; + let getMatchingBuildingBlockAlerts; if (buildingBlockAlerts != null && buildingBlockAlerts.length > 0) - myfunc = (newAlertSource: unknown) => { + getMatchingBuildingBlockAlerts = (newAlertSource: unknown) => { return buildingBlockAlerts?.filter((someAlert) => { - // console.error('SOME ALERT GROUP ID', someAlert?._source[ALERT_GROUP_ID]); - // console.error('NEW ALERT GROUP ID', newAlerts[0]?._source[ALERT_GROUP_ID]); - return ( isEqlBuildingBlockAlert(someAlert?._source) && isEqlShellAlert(newAlertSource) && @@ -123,13 +120,12 @@ export const bulkCreateWithSuppression = async < // `fields` should have already been merged into `doc._source` _source: doc._source, })), - // add building block alerts here when you get back suppressionWindow, enrichAlertsWrapper, alertTimestampOverride, isSuppressionPerRuleExecution, maxAlerts, - myfunc // do the same map as wrappedDocs + getMatchingBuildingBlockAlerts ); const end = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 9b14223125509..eeb8a58b39f9b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -150,7 +150,6 @@ export const wrapSuppressedSequenceAlerts = ({ }): Array> => { // objective here is to replicate what is happening // in x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts - // return sequences.reduce( (acc: Array>, sequence) => { const fields = sequence.events?.reduce( From cbc46e0beee0c4061666507e0971ddd0e1c97aef Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 21:14:34 -0400 Subject: [PATCH 29/70] updates example json for sequence rules --- .../scripts/rules/queries/sequence_eql_query.json | 6 +++--- .../rules/queries/sequence_eql_query_no_duration.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json index ab9af79de9f34..8c5531188379b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -1,5 +1,5 @@ { - "name": "EQL sequence rule", + "name": "EQL sequence duration", "description": "Rule with an eql query", "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], "rule_id": "rule-id-eql-1", @@ -16,7 +16,7 @@ }, "risk_score": 99, "to": "now", - "from": "now-70s", + "from": "now-120s", "severity": "high", "type": "eql", "language": "eql", @@ -55,7 +55,7 @@ "references": ["http://www.example.com/some-article-about-attack"], "alert_suppression": { "group_by": ["agent.name"], - "duration": { "value": 5, "unit": "m" }, + "duration": { "value": 5, "unit": "h" }, "missing_fields_strategy": "suppress" }, "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json index 03a66efe5dc4d..a3df1cfe24938 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json @@ -1,11 +1,11 @@ { - "name": "EQL sequence rule", + "name": "EQL sequence no duration", "description": "Rule with an eql query", "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], "rule_id": "rule-id-eql-2", "enabled": true, "index": ["auditbeat*", "packetbeat*"], - "interval": "2m", + "interval": "30s", "query": "sequence with maxspan=10m [any where agent.type == \"auditbeat\"] [any where event.category == \"network_traffic\"]", "meta": { "anything_you_want_ui_related_or_otherwise": { @@ -16,7 +16,7 @@ }, "risk_score": 99, "to": "now", - "from": "now-5m", + "from": "now-120s", "severity": "high", "type": "eql", "language": "eql", From b9d39505fe8d1a5c89636ac96e59b6f420efd1fc Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 30 Sep 2024 21:16:28 -0400 Subject: [PATCH 30/70] utilize pre-existing logic for building alert in sequence suppression --- .../eql/build_alert_group_from_sequence.ts | 9 +- .../utils/wrap_suppressed_alerts.ts | 111 +++--------------- 2 files changed, 24 insertions(+), 96 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 8fc473c6c7161..c9ed5448c5619 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -47,6 +47,8 @@ export const buildAlertGroupFromSequence = ( buildReasonMessage: BuildReasonMessage, indicesToQuery: string[], alertTimestampOverride: Date | undefined, + applyOverrides = false, + extraFieldsForShellAlert = {}, publicBaseUrl?: string ): Array> => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); @@ -68,7 +70,7 @@ export const buildAlertGroupFromSequence = ( mergeStrategy, ignoreFields: {}, ignoreFieldsRegexes: [], - applyOverrides: false, + applyOverrides, buildReasonMessage, indicesToQuery, alertTimestampOverride, @@ -111,7 +113,10 @@ export const buildAlertGroupFromSequence = ( const sequenceAlert: WrappedFieldsLatest = { _id: shellAlert[ALERT_UUID], _index: '', - _source: shellAlert, + _source: { + ...shellAlert, + ...extraFieldsForShellAlert, + }, }; // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index eeb8a58b39f9b..779a5eb261394 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -7,11 +7,11 @@ import objectHash from 'object-hash'; -import { ALERT_BUILDING_BLOCK_TYPE, ALERT_URL, ALERT_UUID, TIMESTAMP } from '@kbn/rule-data-utils'; +import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import type { Ancestor, SignalSource, SignalSourceHit } from '../types'; +import type { SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, @@ -27,14 +27,9 @@ import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; import { generateId } from './utils'; -import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids'; import type { BuildReasonMessage } from './reason_formatters'; -import { buildAlertRoot } from '../eql/build_alert_group_from_sequence'; -import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_path'; -import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants'; -import { ALERT_GROUP_ID, ALERT_GROUP_INDEX } from '../../../../../common/field_maps/field_names'; -import { buildAncestors } from '../factories/utils/build_alert'; +import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; @@ -148,8 +143,6 @@ export const wrapSuppressedSequenceAlerts = ({ primaryTimestamp: string; secondaryTimestamp?: string; }): Array> => { - // objective here is to replicate what is happening - // in x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts return sequences.reduce( (acc: Array>, sequence) => { const fields = sequence.events?.reduce( @@ -160,100 +153,30 @@ export const wrapSuppressedSequenceAlerts = ({ alertSuppression: completeRule?.ruleParams?.alertSuppression, fields, }); - const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); - if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { - return []; - } - - // The "building block" alerts start out as regular BaseFields. - // We'll add the group ID and index fields - // after creating the shell alert later on - // since that's when the group ID is determined. - const baseAlerts = sequence.events.map((event) => - transformHitToAlert({ - spaceId, - completeRule, - doc: event, - mergeStrategy, - ignoreFields: {}, - ignoreFieldsRegexes: [], - applyOverrides: true, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - ruleExecutionLogger, - alertUuid: 'placeholder-alert-uuid', // This is overriden below, - publicBaseUrl, - }) - ); - const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - // The ID of each building block alert depends on all of the other building blocks as well, - // so we generate the IDs after making all the BaseFields - const buildingBlockIds = generateBuildingBlockIds(baseAlerts); - const wrappedBaseFields: Array> = baseAlerts.map( - (block, i): WrappedFieldsLatest => ({ - _id: buildingBlockIds[i], - _index: '', - _source: { - ...block, - [ALERT_UUID]: buildingBlockIds[i], - }, - }) - ); - - // Now that we have an array of building blocks for the events in the sequence, - // we can build the signal that links the building blocks together - // and also insert the group id (which is also the "shell" signal _id) in each building block - const shellAlert = buildAlertRoot( - wrappedBaseFields, + const alertGroupFromSequence = buildAlertGroupFromSequence( + ruleExecutionLogger, + sequence, completeRule, + mergeStrategy, spaceId, buildReasonMessage, indicesToQuery, alertTimestampOverride, + true, + getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + fields, + suppressionTerms, + fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), + instanceId, + }), publicBaseUrl ); - const sequenceAlert = { - _id: shellAlert[ALERT_UUID], - _index: '', - _source: { - ...shellAlert, - ...getSuppressionAlertFields({ - primaryTimestamp, - secondaryTimestamp, - fields, - suppressionTerms, - fallbackTimestamp: baseAlerts?.[0][TIMESTAMP], - instanceId, - }), - }, - }; - - // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks - const wrappedBuildingBlocks = wrappedBaseFields.map((block, i) => { - const alertUrl = getAlertDetailsUrl({ - alertId: block._id, - index: `${DEFAULT_ALERTS_INDEX}-${spaceId}`, - timestamp: block._source['@timestamp'], - basePath: publicBaseUrl, - spaceId, - }); - - return { - ...block, - _source: { - ...block._source, - [ALERT_BUILDING_BLOCK_TYPE]: 'default', - [ALERT_GROUP_ID]: shellAlert[ALERT_GROUP_ID], - [ALERT_GROUP_INDEX]: i, - [ALERT_URL]: alertUrl, - }, - }; - }); - return [...acc, ...wrappedBuildingBlocks, sequenceAlert] as Array< + return [...acc, ...alertGroupFromSequence] as Array< WrappedFieldsLatest >; }, From 88a6397d67e811da11f2739be1113b254cf19959 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 1 Oct 2024 14:25:38 -0400 Subject: [PATCH 31/70] refactor buildAlertGroupFromSequence to use object param instead of positional params --- .../build_alert_group_from_sequence.test.ts | 20 ++++---- .../eql/build_alert_group_from_sequence.ts | 49 ++++++++++++++----- .../rule_types/eql/wrap_sequences_factory.ts | 6 +-- .../utils/wrap_suppressed_alerts.ts | 10 ++-- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts index 84349e9142e22..a8cb8084e918c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts @@ -43,17 +43,17 @@ describe('buildAlert', () => { sampleDocNoSortId('619389b2-b077-400e-b40b-abde20d675d3'), ], }; - const alertGroup = buildAlertGroupFromSequence( - ruleExecutionLoggerMock, - eqlSequence, + const alertGroup = buildAlertGroupFromSequence({ + ruleExecutionLogger: ruleExecutionLoggerMock, + sequence: eqlSequence, completeRule, - 'allFields', - SPACE_ID, - jest.fn(), - completeRule.ruleParams.index as string[], - undefined, - PUBLIC_BASE_URL - ); + mergeStrategy: 'allFields', + spaceId: SPACE_ID, + buildReasonMessage: jest.fn(), + indicesToQuery: completeRule.ruleParams.index as string[], + alertTimestampOverride: undefined, + publicBaseUrl: PUBLIC_BASE_URL, + }); expect(alertGroup.length).toEqual(3); expect(alertGroup[0]).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index c9ed5448c5619..0c5ccf67208a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -30,6 +30,27 @@ import type { EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; +import type { SuppressionTerm } from '../utils'; + +export interface BuildAlertGroupFromSequence { + ruleExecutionLogger: IRuleExecutionLogForExecutors; + sequence: EqlSequence; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + spaceId: string | null | undefined; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + applyOverrides?: boolean; + extraFieldsForShellAlert?: { + 'kibana.alert.instance.id': string; + 'kibana.alert.suppression.terms': SuppressionTerm[]; + 'kibana.alert.suppression.start': Date; + 'kibana.alert.suppression.end': Date; + 'kibana.alert.suppression.docs_count': number; + }; + publicBaseUrl?: string; +} /** * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - @@ -38,19 +59,21 @@ import type { * @param sequence The raw ES documents that make up the sequence * @param completeRule object representing the rule that found the sequence */ -export const buildAlertGroupFromSequence = ( - ruleExecutionLogger: IRuleExecutionLogForExecutors, - sequence: EqlSequence, - completeRule: CompleteRule, - mergeStrategy: ConfigType['alertMergeStrategy'], - spaceId: string | null | undefined, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, +export const buildAlertGroupFromSequence = ({ + ruleExecutionLogger, + sequence, + completeRule, + mergeStrategy, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, applyOverrides = false, - extraFieldsForShellAlert = {}, - publicBaseUrl?: string -): Array> => { + extraFieldsForShellAlert, + publicBaseUrl, +}: BuildAlertGroupFromSequence): Array< + WrappedFieldsLatest +> => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { return []; @@ -115,7 +138,7 @@ export const buildAlertGroupFromSequence = ( _index: '', _source: { ...shellAlert, - ...extraFieldsForShellAlert, + ...(extraFieldsForShellAlert ?? {}), }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index e21f09438e8b8..75b27bface169 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -39,7 +39,7 @@ export const wrapSequencesFactory = sequences.reduce( (acc: Array>, sequence) => [ ...acc, - ...buildAlertGroupFromSequence( + ...buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, completeRule, @@ -48,8 +48,8 @@ export const wrapSequencesFactory = buildReasonMessage, indicesToQuery, alertTimestampOverride, - publicBaseUrl - ), + publicBaseUrl, + }), ], [] ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 779a5eb261394..4f8d72150a3f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -155,7 +155,7 @@ export const wrapSuppressedSequenceAlerts = ({ }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const alertGroupFromSequence = buildAlertGroupFromSequence( + const alertGroupFromSequence = buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, completeRule, @@ -164,8 +164,8 @@ export const wrapSuppressedSequenceAlerts = ({ buildReasonMessage, indicesToQuery, alertTimestampOverride, - true, - getSuppressionAlertFields({ + applyOverrides: true, + extraFieldsForShellAlert: getSuppressionAlertFields({ primaryTimestamp, secondaryTimestamp, fields, @@ -173,8 +173,8 @@ export const wrapSuppressedSequenceAlerts = ({ fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), instanceId, }), - publicBaseUrl - ); + publicBaseUrl, + }); return [...acc, ...alertGroupFromSequence] as Array< WrappedFieldsLatest From d0ad7141ff1a7de1b5926a02284144a794be057c Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 1 Oct 2024 14:52:30 -0400 Subject: [PATCH 32/70] add comment for feature flag --- .../security_solution/common/experimental_features.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 33997e9d1c104..31000e545b352 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -12,8 +12,14 @@ export type ExperimentalFeatures = { [K in keyof typeof allowedExperimentalValue * This object is then used to validate and parse the value entered. */ export const allowedExperimentalValues = Object.freeze({ - /** - * feature flag for eql sequence alert suppression + /* + * Enables experimental feature flag for eql sequence alert suppression. + * + * Ticket: https://github.com/elastic/security-team/issues/9608 + * Owners: https://github.com/orgs/elastic/teams/security-detection-engine + * Added: on October 1st, 2024 in https://github.com/elastic/kibana/pull/189725 + * Turned: on (TBD) + * Expires: on (TBD) */ alertSuppressionForSequenceEqlRuleEnabled: false, From b9ed401f16686445eabe601b2caf03fb982d374b Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 14 Oct 2024 14:09:03 -0400 Subject: [PATCH 33/70] update logic for building block alerts --- .../server/utils/create_persistence_rule_type_wrapper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index c40331379aa0b..7a5d528b1e2b6 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -602,7 +602,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper : []; const augmentedBuildingBlockAlerts = - getMatchingBuildingBlockAlerts != null && newAlerts?.length > 0 + matchingBuildingBlockAlerts.length > 0 ? await augmentAlerts({ alerts: matchingBuildingBlockAlerts, options, From 1709a05f8125e1991f465eac689932865824d245 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 25 Oct 2024 15:56:29 -0400 Subject: [PATCH 34/70] move logic for suppression terms and fields to after we build shell alert --- .../create_persistence_rule_type_wrapper.ts | 1 - .../eql/build_alert_group_from_sequence.ts | 70 +++++++++++-------- ...bulk_create_suppressed_alerts_in_memory.ts | 8 +-- .../rule_types/utils/suppression_utils.ts | 3 +- .../utils/wrap_suppressed_alerts.ts | 51 ++++++++------ 5 files changed, 78 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index f116304bd820b..d879aac1842d1 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -590,7 +590,6 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, - intendedTimestamp, }) : []; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 63c26f647124f..30d03be8459a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -5,6 +5,13 @@ * 2.0. */ +import type { + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, +} from '@kbn/rule-data-utils'; import { ALERT_URL, ALERT_UUID } from '@kbn/rule-data-utils'; import { intersection as lodashIntersection, isArray } from 'lodash'; @@ -32,6 +39,14 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import type { SuppressionTerm } from '../utils'; +export interface ExtraFieldsForShellAlert { + [ALERT_INSTANCE_ID]: string; + [ALERT_SUPPRESSION_TERMS]: SuppressionTerm[]; + [ALERT_SUPPRESSION_START]: Date; + [ALERT_SUPPRESSION_END]: Date; + [ALERT_SUPPRESSION_DOCS_COUNT]: number; +} + export interface BuildAlertGroupFromSequence { ruleExecutionLogger: IRuleExecutionLogForExecutors; sequence: EqlSequence; @@ -42,13 +57,6 @@ export interface BuildAlertGroupFromSequence { indicesToQuery: string[]; alertTimestampOverride: Date | undefined; applyOverrides?: boolean; - extraFieldsForShellAlert?: { - 'kibana.alert.instance.id': string; - 'kibana.alert.suppression.terms': SuppressionTerm[]; - 'kibana.alert.suppression.start': Date; - 'kibana.alert.suppression.end': Date; - 'kibana.alert.suppression.docs_count': number; - }; publicBaseUrl?: string; intendedTimestamp?: Date; } @@ -69,8 +77,6 @@ export const buildAlertGroupFromSequence = ({ buildReasonMessage, indicesToQuery, alertTimestampOverride, - applyOverrides = false, - extraFieldsForShellAlert, publicBaseUrl, intendedTimestamp, }: BuildAlertGroupFromSequence): Array< @@ -95,7 +101,7 @@ export const buildAlertGroupFromSequence = ({ mergeStrategy, ignoreFields: {}, ignoreFieldsRegexes: [], - applyOverrides, + applyOverrides: false, buildReasonMessage, indicesToQuery, alertTimestampOverride, @@ -127,23 +133,20 @@ export const buildAlertGroupFromSequence = ({ // Now that we have an array of building blocks for the events in the sequence, // we can build the signal that links the building blocks together // and also insert the group id (which is also the "shell" signal _id) in each building block - const shellAlert = buildAlertRoot( - wrappedBaseFields, + const shellAlert = buildAlertRoot({ + wrappedBuildingBlocks: wrappedBaseFields, completeRule, spaceId, buildReasonMessage, indicesToQuery, alertTimestampOverride, publicBaseUrl, - intendedTimestamp - ); + intendedTimestamp, + }); const sequenceAlert: WrappedFieldsLatest = { _id: shellAlert[ALERT_UUID], _index: '', - _source: { - ...shellAlert, - ...(extraFieldsForShellAlert ?? {}), - }, + _source: shellAlert, }; // Finally, we have the group id from the shell alert so we can convert the BaseFields into EqlBuildingBlocks @@ -173,16 +176,27 @@ export const buildAlertGroupFromSequence = ({ return [...wrappedBuildingBlocks, sequenceAlert]; }; -export const buildAlertRoot = ( - wrappedBuildingBlocks: Array>, - completeRule: CompleteRule, - spaceId: string | null | undefined, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, - publicBaseUrl?: string, - intendedTimestamp?: Date -): EqlShellFieldsLatest => { +export interface BuildAlertRootParams { + wrappedBuildingBlocks: Array>; + completeRule: CompleteRule; + spaceId: string | null | undefined; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + publicBaseUrl?: string; + intendedTimestamp?: Date; +} + +export const buildAlertRoot = ({ + wrappedBuildingBlocks, + completeRule, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl, + intendedTimestamp, +}: BuildAlertRootParams): EqlShellFieldsLatest => { const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source)); const reason = buildReasonMessage({ name: completeRule.ruleConfig.name, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index c4c79d9c620a1..9fb9f8755beb6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -8,7 +8,6 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import { ALERT_BUILDING_BLOCK_TYPE } from '@kbn/rule-data-utils'; import partition from 'lodash/partition'; import type { @@ -21,7 +20,7 @@ import type { WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; -import { addToSearchAfterReturn } from './utils'; +import { addToSearchAfterReturn, isEqlBuildingBlockAlert } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; @@ -206,9 +205,8 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ ); // partition sequence alert from building block alerts - const [sequenceAlerts, buildingBlockAlerts] = partition( - suppressibleWrappedDocs, - (signal) => signal._source[ALERT_BUILDING_BLOCK_TYPE] == null + const [buildingBlockAlerts, sequenceAlerts] = partition(suppressibleWrappedDocs, (signal) => + isEqlBuildingBlockAlert(signal._source) ); return executeBulkCreateAlerts({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 175a629c2985c..552bac024b406 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -17,6 +17,7 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; +import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; export interface SuppressionTerm { field: string; @@ -47,7 +48,7 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp ); - const suppressionFields = { + const suppressionFields: ExtraFieldsForShellAlert = { [ALERT_INSTANCE_ID]: instanceId, [ALERT_SUPPRESSION_TERMS]: suppressionTerms, [ALERT_SUPPRESSION_START]: suppressionTime, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 0af61b0e0ada1..19ff19e71bae6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -11,6 +11,7 @@ import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; +import partition from 'lodash/partition'; import type { SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, @@ -26,9 +27,10 @@ import type { import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { generateId } from './utils'; +import { generateId, isEqlBuildingBlockAlert } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; +import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; @@ -148,16 +150,6 @@ export const wrapSuppressedSequenceAlerts = ({ }): Array> => { return sequences.reduce( (acc: Array>, sequence) => { - const fields = sequence.events?.reduce( - (seqAcc, event) => ({ ...seqAcc, ...event.fields }), - {} - ); - const suppressionTerms = getSuppressionTerms({ - alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields, - }); - const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const alertGroupFromSequence = buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, @@ -168,18 +160,37 @@ export const wrapSuppressedSequenceAlerts = ({ indicesToQuery, alertTimestampOverride, applyOverrides: true, - extraFieldsForShellAlert: getSuppressionAlertFields({ - primaryTimestamp, - secondaryTimestamp, - fields, - suppressionTerms, - fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), - instanceId, - }), publicBaseUrl, }); - return [...acc, ...alertGroupFromSequence] as Array< + // find shell alert + const [buildingBlocks, shellAlert] = partition(alertGroupFromSequence, (alert) => + isEqlBuildingBlockAlert(alert._source) + ); + + console.error('BUILDING BLOCKS LENGTH', buildingBlocks.length); + console.error('SHELL ALLERT LENGTH', shellAlert.length); + + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields: shellAlert[0]?._source, + }); + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const suppressionFields = getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + // as casting should work because the alert fields are flattened (hopefully?) + fields: shellAlert[0]._source as Record | undefined, + suppressionTerms, + fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), + instanceId, + }); + const theFields = Object.keys(suppressionFields) as Array; + // mutates shell alert to contain values from suppression fields + theFields.forEach((field) => (shellAlert[0]._source[field] = suppressionFields[field])); + + return [...acc, ...buildingBlocks, ...shellAlert] as Array< WrappedFieldsLatest >; }, From 36f9b975813a279d1454214f09566cd9309e64b4 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Sun, 27 Oct 2024 18:22:24 -0400 Subject: [PATCH 35/70] prevent eql sequence suppression for building block rule types --- .../create_rule/request_schema_validation.ts | 13 ++++++++++++ .../patch_rule/request_schema_validation.ts | 15 ++++++++++++++ .../update_rule/request_schema_validation.ts | 13 ++++++++++++ .../import_rules/rule_to_import_validation.ts | 19 +++++++++++++++++- .../components/step_about_rule/index.tsx | 20 ++++++++++++++++++- .../pages/rule_creation/index.tsx | 11 ++++++++++ .../pages/rule_editing/index.tsx | 12 ++++++++++- .../detection_engine/rule_types/eql/eql.ts | 2 ++ 8 files changed, 102 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts index 519ef874c422e..db8ccf3eddbe1 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts @@ -5,6 +5,7 @@ * 2.0. */ +import isEmpty from 'lodash/isEmpty'; import type { RuleCreateProps } from '../../../model'; /** @@ -16,9 +17,21 @@ export const validateCreateRuleProps = (props: RuleCreateProps): string[] => { ...validateTimelineTitle(props), ...validateThreatMapping(props), ...validateThreshold(props), + ...validateSequenceSuppressionXorBuildingBlockRuleType(props), ]; }; +const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleCreateProps): string[] => { + if ( + props.type === 'eql' && + !isEmpty(props.building_block_type) && + !isEmpty(props.alert_suppression) + ) { + return ['rule cannot be an eql rule with building block type and have suppression enabled']; + } + return []; +}; + const validateTimelineId = (props: RuleCreateProps): string[] => { if (props.timeline_id != null) { if (props.timeline_title == null) { diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts index cb34c7fa8ecb5..e3686e1d19d06 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts @@ -5,6 +5,7 @@ * 2.0. */ +import isEmpty from 'lodash/isEmpty'; import type { PatchRuleRequestBody } from './patch_rule_route.gen'; /** @@ -16,9 +17,23 @@ export const validatePatchRuleRequestBody = (rule: PatchRuleRequestBody): string ...validateTimelineId(rule), ...validateTimelineTitle(rule), ...validateThreshold(rule), + ...validateSequenceSuppressionXorBuildingBlockRuleType(rule), ]; }; +const validateSequenceSuppressionXorBuildingBlockRuleType = ( + props: PatchRuleRequestBody +): string[] => { + if ( + props.type === 'eql' && + !isEmpty(props.building_block_type) && + !isEmpty(props.alert_suppression) + ) { + return ['rule cannot be an eql rule with building block type and have suppression enabled']; + } + return []; +}; + const validateId = (rule: PatchRuleRequestBody): string[] => { if (rule.id != null && rule.rule_id != null) { return ['both "id" and "rule_id" cannot exist, choose one or the other']; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts index d58bbdc4bee05..a5c85d30a6c8b 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts @@ -5,6 +5,7 @@ * 2.0. */ +import isEmpty from 'lodash/isEmpty'; import type { RuleUpdateProps } from '../../../model'; /** @@ -16,9 +17,21 @@ export const validateUpdateRuleProps = (props: RuleUpdateProps): string[] => { ...validateTimelineId(props), ...validateTimelineTitle(props), ...validateThreshold(props), + ...validateSequenceSuppressionXorBuildingBlockRuleType(props), ]; }; +const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleUpdateProps): string[] => { + if ( + props.type === 'eql' && + !isEmpty(props.building_block_type) && + !isEmpty(props.alert_suppression) + ) { + return ['rule cannot be an eql rule with building block type and have suppression enabled']; + } + return []; +}; + const validateId = (props: RuleUpdateProps): string[] => { if (props.id != null && props.rule_id != null) { return ['both "id" and "rule_id" cannot exist, choose one or the other']; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts index de21ac3a7964c..352ab15afbf15 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts @@ -5,13 +5,30 @@ * 2.0. */ +import isEmpty from 'lodash/isEmpty'; import type { RuleToImport } from './rule_to_import'; /** * Additional validation that is implemented outside of the schema itself. */ export const validateRuleToImport = (rule: RuleToImport): string[] => { - return [...validateTimelineId(rule), ...validateTimelineTitle(rule), ...validateThreshold(rule)]; + return [ + ...validateTimelineId(rule), + ...validateTimelineTitle(rule), + ...validateThreshold(rule), + ...validateSequenceSuppressionXorBuildingBlockRuleType(rule), + ]; +}; + +const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleToImport): string[] => { + if ( + props.type === 'eql' && + !isEmpty(props.building_block_type) && + !isEmpty(props.alert_suppression) + ) { + return ['rule cannot be an eql rule with building block type and have suppression enabled']; + } + return []; }; const validateTimelineId = (rule: RuleToImport): string[] => { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx index 7666a9ba8aee3..972ceb93c4b34 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx @@ -55,6 +55,7 @@ interface StepAboutRuleProps extends RuleStepProps { timestampOverride: string; form: FormHook; esqlQuery?: string | undefined; + eqlSequenceSuppressionEnabled: boolean; } interface StepAboutRuleReadOnlyProps { @@ -85,6 +86,7 @@ const StepAboutRuleComponent: FC = ({ isLoading, form, esqlQuery, + eqlSequenceSuppressionEnabled, }) => { const { data } = useKibana().services; @@ -93,6 +95,16 @@ const StepAboutRuleComponent: FC = ({ const { ruleIndices } = useRuleIndices(machineLearningJobId, index); + const [buildingBlockRuleTypeChecked, setBuildingBlockRuleTypeChecked] = useState(false); + + const buildingBlockRuleTypeCheckedOnChange = (e: React.ChangeEvent) => { + if (eqlSequenceSuppressionEnabled) { + setBuildingBlockRuleTypeChecked(false); + } else { + setBuildingBlockRuleTypeChecked(e.target.checked); + } + }; + /** * 1. if not null, fetch data view from id saved on rule form * 2. Create a state to set the indexPattern to be used @@ -320,13 +332,19 @@ const StepAboutRuleComponent: FC = ({ /> + {/* do not display building block rule type option if eql sequence suppression is in use*/} ) => + buildingBlockRuleTypeCheckedOnChange(e), }, }} /> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx index 500fedb4d0005..462fa735f2760 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx @@ -26,6 +26,7 @@ import { isMlRule, isThreatMatchRule, isEsqlRule, + isEqlSequenceQuery, } from '../../../../../common/detection_engine/utils'; import { useCreateRule } from '../../../rule_management/logic'; import type { RuleCreateProps } from '../../../../../common/api/detection_engine/model/rule_schema'; @@ -83,6 +84,7 @@ import { NextStep } from '../../components/next_step'; import { useRuleForms, useRuleFormsErrors, useRuleIndexPattern } from '../form'; import { CustomHeaderPageMemo } from '..'; import { SaveWithErrorsModal } from '../../components/save_with_errors_confirmation'; +import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; const MyEuiPanel = styled(EuiPanel)<{ zindex?: number; @@ -219,6 +221,12 @@ const CreateRulePageComponent: React.FC = () => { const esqlQueryForAboutStep = useEsqlQueryForAboutStep({ defineStepData, activeStep }); + const isEqlSeqQuery = isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string); + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( + ruleType, + isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string) + ); + const esqlIndex = useEsqlIndex(defineStepData.queryBar.query.query, ruleType); const memoizedIndex = useMemo( @@ -679,6 +687,7 @@ const CreateRulePageComponent: React.FC = () => { isLoading={isCreateRuleLoading || loading} form={aboutStepForm} esqlQuery={esqlQueryForAboutStep} + eqlSequenceSuppressionEnabled={isAlertSuppressionEnabled && isEqlSeqQuery} /> { loading, memoAboutStepReadOnly, esqlQueryForAboutStep, + isAlertSuppressionEnabled, + isEqlSeqQuery, ] ); const memoAboutStepExtraAction = useMemo( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index 88b080170da6a..ae0e1cd6da19d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -24,7 +24,7 @@ import { useParams } from 'react-router-dom'; import type { DataViewListItem } from '@kbn/data-views-plugin/common'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; -import { isEsqlRule } from '../../../../../common/detection_engine/utils'; +import { isEqlSequenceQuery, isEsqlRule } from '../../../../../common/detection_engine/utils'; import { RulePreview } from '../../components/rule_preview'; import { getIsRulePreviewDisabled } from '../../components/rule_preview/helpers'; import type { @@ -73,6 +73,7 @@ import { useRuleForms, useRuleFormsErrors, useRuleIndexPattern } from '../form'; import { useEsqlIndex, useEsqlQueryForAboutStep } from '../../hooks'; import { CustomHeaderPageMemo } from '..'; import { SaveWithErrorsModal } from '../../components/save_with_errors_confirmation'; +import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const { addSuccess } = useAppToasts(); @@ -161,6 +162,12 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const esqlIndex = useEsqlIndex(defineStepData.queryBar.query.query, defineStepData.ruleType); + const isEqlSeqQuery = isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string); + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( + defineStepData.ruleType, + isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string) + ); + const memoizedIndex = useMemo( () => (isEsqlRule(defineStepData.ruleType) ? esqlIndex : defineStepData.index), [defineStepData.index, esqlIndex, defineStepData.ruleType] @@ -298,6 +305,7 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { form={aboutStepForm} esqlQuery={esqlQueryForAboutStep} key="aboutStep" + eqlSequenceSuppressionEnabled={isAlertSuppressionEnabled && isEqlSeqQuery} /> )} @@ -389,6 +397,8 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { actionsStepForm, memoizedIndex, esqlQueryForAboutStep, + isAlertSuppressionEnabled, + isEqlSeqQuery, ] ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 8ece7e9d55f4d..1c89fa02540ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -111,6 +111,8 @@ export const eqlExecutor = async ({ const isLoggedRequestsEnabled = state?.isLoggedRequestsEnabled ?? false; const loggedRequests: RulePreviewLoggedRequest[] = []; + // TODO: fix complexity warning + // eslint-disable-next-line complexity return withSecuritySpan('eqlExecutor', async () => { const result = createSearchAfterReturnType(); From 926478fe182961de220e853218a29731fb74c3b8 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 4 Nov 2024 16:13:07 -0500 Subject: [PATCH 36/70] add subAlerts when wrapping suppressed sequences to be used in alertWithSuppression --- .../create_persistence_rule_type_wrapper.ts | 20 +- .../server/utils/persistence_types.ts | 7 +- .../components/step_about_rule/index.test.tsx | 1 + .../build_alert_group_from_sequence.test.ts | 6 +- .../eql/build_alert_group_from_sequence.ts | 21 +- .../rule_types/eql/wrap_sequences_factory.ts | 23 +- .../lib/detection_engine/rule_types/types.ts | 12 +- ...bulk_create_suppressed_alerts_in_memory.ts | 13 +- .../utils/bulk_create_with_suppression.ts | 25 +- .../utils/wrap_suppressed_alerts.ts | 54 ++- .../execution_logic/eql_alert_suppression.ts | 418 ++++++++++++------ 11 files changed, 398 insertions(+), 202 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index d879aac1842d1..e820dcc2369e5 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -371,8 +371,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper enrichAlerts, currentTimeOverride, isRuleExecutionOnly, - maxAlerts, - getMatchingBuildingBlockAlerts + maxAlerts ) => { const ruleDataClientWriter = await ruleDataClient.getWriter({ namespace: options.spaceId, @@ -572,21 +571,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); - const matchingBuildingBlockAlerts = - newAlerts?.length > 0 && getMatchingBuildingBlockAlerts != null - ? newAlerts.flatMap( - (newAlert) => - getMatchingBuildingBlockAlerts(newAlert?._source) as Array<{ - _id: string; - _source: Record; - }> - ) - : []; - const augmentedBuildingBlockAlerts = - matchingBuildingBlockAlerts.length > 0 + newAlerts != null && newAlerts.length > 0 ? await augmentAlerts({ - alerts: matchingBuildingBlockAlerts, + alerts: newAlerts.flatMap((alert) => + alert.subAlerts != null ? alert.subAlerts : [] + ), options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index caf1df5551128..f6e1ae5942b37 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -45,6 +45,10 @@ export type SuppressedAlertService = ( alerts: Array<{ _id: string; _source: T; + subAlerts?: Array<{ + _id: string; + _source: T; + }>; }>, suppressionWindow: string, enrichAlerts?: ( @@ -53,8 +57,7 @@ export type SuppressedAlertService = ( ) => Promise>, currentTimeOverride?: Date, isRuleExecutionOnly?: boolean, - maxAlerts?: number, - getMatchingBuildingBlockAlerts?: (alert: unknown) => unknown[] + maxAlerts?: number ) => Promise>; export interface SuppressedAlertServiceResult extends PersistenceAlertServiceResult { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx index bdbc01ada58ff..9364764c7fd55 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx @@ -110,6 +110,7 @@ describe('StepAboutRuleComponent', () => { timestampOverride={stepAboutDefaultValue.timestampOverride} isLoading={false} form={aboutStepForm} + eqlSequenceSuppressionEnabled={false} /> ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts index a8cb8084e918c..70778385396c0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts @@ -72,7 +72,7 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[0]._source[ALERT_URL]).toContain( + expect(alertGroup[0]?._source?.[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/f2db3574eaf8450e3f4d1cf4f416d70b110b035ae0a7a00026242df07f0a6c90?index=.alerts-security.alerts-space' ); expect(alertGroup[1]).toEqual( @@ -113,7 +113,7 @@ describe('buildAlert', () => { }, { depth: 1, - id: alertGroup[0]._id, + id: alertGroup[0]?._id, index: '', rule: sampleRuleGuid, type: 'signal', @@ -135,7 +135,7 @@ describe('buildAlert', () => { expect(alertGroup[2]._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1b7d06954e74257140f3bf73f139078483f9658fe829fd806cc307fc0388fb23?index=.alerts-security.alerts-space' ); - const groupIds = alertGroup.map((alert) => alert._source[ALERT_GROUP_ID]); + const groupIds = alertGroup.map((alert) => alert?._source?.[ALERT_GROUP_ID]); for (const groupId of groupIds) { expect(groupId).toEqual(groupIds[0]); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 30d03be8459a6..2a7f4c291fd0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -61,6 +61,17 @@ export interface BuildAlertGroupFromSequence { intendedTimestamp?: Date; } +// eql shell alerts can have a subAlerts property +// when suppression is used in EQL sequence queries +export type WrappedEqlShellOptionalSubAlertsType = WrappedFieldsLatest & { + subAlerts?: Array>; +}; + +export type BuildAlertGroupFromSequenceReturnType = [ + WrappedEqlShellOptionalSubAlertsType?, + ...Array> +]; + /** * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals @@ -79,9 +90,7 @@ export const buildAlertGroupFromSequence = ({ alertTimestampOverride, publicBaseUrl, intendedTimestamp, -}: BuildAlertGroupFromSequence): Array< - WrappedFieldsLatest -> => { +}: BuildAlertGroupFromSequence): BuildAlertGroupFromSequenceReturnType => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { return []; @@ -173,7 +182,11 @@ export const buildAlertGroupFromSequence = ({ } ); - return [...wrappedBuildingBlocks, sequenceAlert]; + // sequence alert guaranteed to be first + return [sequenceAlert, ...wrappedBuildingBlocks] as [ + WrappedFieldsLatest?, + ...Array> + ]; }; export interface BuildAlertRootParams { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index 2ddf8c268562e..2aa77bcc2224d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -11,7 +11,8 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, RuleParams } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import type { - BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; @@ -39,9 +40,14 @@ export const wrapSequencesFactory = }): WrapSequences => (sequences, buildReasonMessage) => sequences.reduce( - (acc: Array>, sequence) => [ - ...acc, - ...buildAlertGroupFromSequence({ + ( + acc: Array< + | WrappedFieldsLatest + | WrappedFieldsLatest + >, + sequence + ) => { + const alerts = buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, completeRule, @@ -52,7 +58,12 @@ export const wrapSequencesFactory = alertTimestampOverride, publicBaseUrl, intendedTimestamp, - }), - ], + }); + + return [...acc, ...alerts] as Array< + | WrappedFieldsLatest + | WrappedFieldsLatest + >; + }, [] ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 4094d6f46fdcb..4d2d4914e6314 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -52,6 +52,8 @@ import type { BuildReasonMessage } from './utils/reason_formatters'; import type { BaseFieldsLatest, DetectionAlert, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../common/api/detection_engine/model/alerts'; import type { @@ -354,12 +356,18 @@ export type WrapSuppressedHits = ( export type WrapSuppressedSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage -) => Array>; +) => Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } +>; export type WrapSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage -) => Array>; +) => Array< + WrappedFieldsLatest | WrappedFieldsLatest +>; export type RuleServices = RuleExecutorServices< AlertInstanceState, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 9fb9f8755beb6..cf07780f0a599 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -8,7 +8,6 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import partition from 'lodash/partition'; import type { SearchAfterAndBulkCreateParams, @@ -20,7 +19,7 @@ import type { WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; -import { addToSearchAfterReturn, isEqlBuildingBlockAlert } from './utils'; +import { addToSearchAfterReturn } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; @@ -199,20 +198,14 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ suppressibleSequences = sequences; } - const suppressibleWrappedDocs = wrapSuppressedSequences( + const suppressibleWrappedDocsWithSubAlerts = wrapSuppressedSequences( suppressibleSequences, buildReasonMessage ); - // partition sequence alert from building block alerts - const [buildingBlockAlerts, sequenceAlerts] = partition(suppressibleWrappedDocs, (signal) => - isEqlBuildingBlockAlert(signal._source) - ); - return executeBulkCreateAlerts({ - suppressibleWrappedDocs: sequenceAlerts, + suppressibleWrappedDocs: suppressibleWrappedDocsWithSubAlerts, unsuppressibleWrappedDocs, - buildingBlockAlerts, toReturn, bulkCreate, services, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 5337e4c6ce376..e88ff2ec0bf71 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -14,10 +14,9 @@ import type { SuppressionFieldsLatest, } from '@kbn/rule-registry-plugin/common/schemas'; -import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; import { isQueryRule } from '../../../../../common/detection_engine/utils'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { makeFloatString, isEqlBuildingBlockAlert, isEqlShellAlert } from './utils'; +import { makeFloatString } from './utils'; import type { BaseFieldsLatest, WrappedFieldsLatest, @@ -55,7 +54,7 @@ export const bulkCreateWithSuppression = async < }: { alertWithSuppression: SuppressedAlertService; ruleExecutionLogger: IRuleExecutionLogForExecutors; - wrappedDocs: Array>; + wrappedDocs: Array & { subAlerts?: Array> }>; buildingBlockAlerts?: Array>; services: RuleServices; suppressionWindow: string; @@ -100,32 +99,22 @@ export const bulkCreateWithSuppression = async < } }; - let getMatchingBuildingBlockAlerts; - - if (buildingBlockAlerts != null && buildingBlockAlerts.length > 0) - getMatchingBuildingBlockAlerts = (newAlertSource: unknown) => { - return buildingBlockAlerts?.filter((someAlert) => { - return ( - isEqlBuildingBlockAlert(someAlert?._source) && - isEqlShellAlert(newAlertSource) && - someAlert?._source?.[ALERT_GROUP_ID] === newAlertSource?.[ALERT_GROUP_ID] - ); - }); - }; - const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( wrappedDocs.map((doc) => ({ _id: doc._id, // `fields` should have already been merged into `doc._source` _source: doc._source, + subAlerts: + doc?.subAlerts != null + ? doc?.subAlerts?.map((subAlert) => ({ _id: subAlert._id, _source: subAlert._source })) + : undefined, })), suppressionWindow, enrichAlertsWrapper, alertTimestampOverride, isSuppressionPerRuleExecution, - maxAlerts, - getMatchingBuildingBlockAlerts + maxAlerts ); const end = performance.now(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 19ff19e71bae6..147e029fe92bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -11,11 +11,11 @@ import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import partition from 'lodash/partition'; import type { SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, + EqlBuildingBlockFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; import type { @@ -27,7 +27,7 @@ import type { import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { generateId, isEqlBuildingBlockAlert } from './utils'; +import { generateId, isEqlShellAlert } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; @@ -147,9 +147,20 @@ export const wrapSuppressedSequenceAlerts = ({ publicBaseUrl: string | undefined; primaryTimestamp: string; secondaryTimestamp?: string; -}): Array> => { +}): Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } +> => { return sequences.reduce( - (acc: Array>, sequence) => { + ( + acc: Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } + >, + sequence + ) => { const alertGroupFromSequence = buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, @@ -162,18 +173,16 @@ export const wrapSuppressedSequenceAlerts = ({ applyOverrides: true, publicBaseUrl, }); - - // find shell alert - const [buildingBlocks, shellAlert] = partition(alertGroupFromSequence, (alert) => - isEqlBuildingBlockAlert(alert._source) - ); - - console.error('BUILDING BLOCKS LENGTH', buildingBlocks.length); - console.error('SHELL ALLERT LENGTH', shellAlert.length); - + const shellAlert = alertGroupFromSequence[0]; + if (!isEqlShellAlert(shellAlert)) { + return [...acc]; + } + const buildingBlocks = alertGroupFromSequence.slice(1) as Array< + WrappedFieldsLatest + >; const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: shellAlert[0]?._source, + fields: shellAlert?._source, }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); @@ -181,19 +190,26 @@ export const wrapSuppressedSequenceAlerts = ({ primaryTimestamp, secondaryTimestamp, // as casting should work because the alert fields are flattened (hopefully?) - fields: shellAlert[0]._source as Record | undefined, + fields: shellAlert?._source as Record | undefined, suppressionTerms, fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), instanceId, }); const theFields = Object.keys(suppressionFields) as Array; // mutates shell alert to contain values from suppression fields - theFields.forEach((field) => (shellAlert[0]._source[field] = suppressionFields[field])); + theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); + shellAlert.subAlerts = buildingBlocks; - return [...acc, ...buildingBlocks, ...shellAlert] as Array< - WrappedFieldsLatest + return [...acc, shellAlert] as Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } >; }, - [] as Array> + [] as Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } + > ); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts index badb72f62e6ca..e01bc2b4e0aa9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql_alert_suppression.ts @@ -1843,7 +1843,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - // we expect one alert and two suppressed alerts + // we expect one created alert and one suppressed alert // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( @@ -1858,13 +1858,11 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp2, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -1896,7 +1894,12 @@ export default ({ getService }: FtrProviderContext) => { // in one of the two events in the sequence, the sequence alert will // adopt the value for host.name and be suppressible. - await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithNoHost, + { ...doc2WithNoHost, '@timestamp': '2020-10-28T06:53:01.000Z' }, + ]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), @@ -1920,28 +1923,38 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - // we expect one alert and two suppressed alerts + // we expect one alert and one suppressed alerts // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( + expect(previewAlerts.length).toEqual(6); + const [sequenceAlerts, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlerts.length).toEqual(2); - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, + expect(sequenceAlerts[0]?._source).toEqual({ + ...sequenceAlerts[0]?._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + expect(sequenceAlerts[1]?._source).toEqual({ + ...sequenceAlerts[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp2, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -1998,27 +2011,37 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(3); + expect(previewAlerts.length).toEqual(6); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(2); - expect(sequenceAlert.length).toEqual(1); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); expect(sequenceAlert[0]?._source).toEqual({ ...sequenceAlert[0]?._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp2, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + expect(sequenceAlert[1]?._source).toEqual({ + ...sequenceAlert[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }); }); @@ -2075,22 +2098,18 @@ export default ({ getService }: FtrProviderContext) => { previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + ); expect(buildingBlockAlerts.length).toEqual(4); expect(sequenceAlert.length).toEqual(2); - - expect(sequenceAlert[1]?._source).toEqual({ - ...sequenceAlert[1]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], + expect(suppressedSequenceAlerts.length).toEqual(0); + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: undefined, [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + [ALERT_SUPPRESSION_DOCS_COUNT]: undefined, }); }); @@ -2150,16 +2169,21 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(6); + expect(previewAlerts.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(4); - expect(sequenceAlert.length).toEqual(2); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); - expect(sequenceAlert[1]?._source).toEqual({ - ...sequenceAlert[1]?._source, + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', @@ -2168,13 +2192,18 @@ export default ({ getService }: FtrProviderContext) => { ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp2, - [ALERT_SUPPRESSION_END]: laterTimestamp3, - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); }); it('does not suppress alerts when "doNotSuppress" is set and suppression field value is undefined for a sequence alert in a given rule execution', async () => { + // This logic should be understood within the confines of the + // objectPairIntersection function defined in + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + // Any sequence comprised of events where one field contains a value, followed by any number of + // events in the sequence where the value is null or undefined will have that field + // stripped from the sequence alert. So given that, we expect three alerts here + // all with the suppression value as undefined const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; const laterTimestamp = '2020-10-28T06:51:00.000Z'; @@ -2204,10 +2233,6 @@ export default ({ getService }: FtrProviderContext) => { host: { name: undefined }, }; - // first suppressible sequence alert will be doc1, doc1WithNoHost - // two unsuppressible sequence alerts will consist of - // [doc1WithNoHost, doc2WithNoHost] and [doc2WithNoHost, doc3WithNoHost] - await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); const rule: EqlRuleCreateProps = { @@ -2232,30 +2257,27 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - // we expect two unsuppressed alerts and one suppressed alert - // for a total of 3 sequence alerts and two building block alerts per alert - // for a total of 6 building block alerts. Let's confirm that + expect(previewAlerts.length).toEqual(9); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + ); + // no alerts should be suppressed because doNotSuppress is set + expect(suppressedSequenceAlerts.length).toEqual(0); expect(buildingBlockAlerts.length).toEqual(6); + // 3 sequence alerts comprised of + // (doc1 + doc1WithNoHost), (doc1WithNoHost + doc2WithNoHost), (doc2WithNoHost + doc3WithNoHost) expect(sequenceAlert.length).toEqual(3); - expect(sequenceAlert[2]?._source).toEqual({ - ...sequenceAlert[2]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: ['host-a'], - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: laterTimestamp, - [ALERT_SUPPRESSION_END]: laterTimestamp, - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: undefined, + [ALERT_SUPPRESSION_DOCS_COUNT]: undefined, }); }); @@ -2263,8 +2285,8 @@ export default ({ getService }: FtrProviderContext) => { const id = uuidv4(); const timestamp = '2020-10-28T06:05:00.000Z'; // this should not count towards events const laterTimestamp = '2020-10-28T06:50:00.000Z'; - const timestamp1 = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:00.000Z'; const doc1 = { id, '@timestamp': timestamp, @@ -2274,12 +2296,14 @@ export default ({ getService }: FtrProviderContext) => { ...doc1, '@timestamp': laterTimestamp, }; + const doc2WithLaterTimestamp = { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }; + const doc3WithLaterTimestamp = { ...doc2WithLaterTimestamp, '@timestamp': laterTimestamp3 }; await indexListOfSourceDocuments([ doc1, doc1WithLaterTimestamp, - { ...doc1, '@timestamp': timestamp1 }, - { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }, + doc2WithLaterTimestamp, + doc3WithLaterTimestamp, ]); const rule: EqlRuleCreateProps = { @@ -2319,23 +2343,29 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp1, - [ALERT_SUPPRESSION_END]: laterTimestamp2, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); - it('does not suppress alerts where no suppression field values match', async () => { + it('suppresses sequence alert with suppression value of "null" when all events have unique values for suppression field', async () => { + /* + Sequence alerts only contain values which make up the intersection of + a given field in all events for that sequence. + + So for a sequence alert where the two events contain host name values of + host-a and host-b, the sequence alert will have 'null' for that value and will + suppress on null values if missing_fields_strategy is set to 'suppress' + */ const id = uuidv4(); - const timestamp = '2020-10-28T06:50:00.000Z'; // this should not count towards events + const timestamp = '2020-10-28T06:50:00.000Z'; const laterTimestamp = '2020-10-28T06:50:01.000Z'; - const timestamp1 = '2020-10-28T06:51:00.000Z'; - const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:00.000Z'; const doc1 = { id, '@timestamp': timestamp, @@ -2346,12 +2376,21 @@ export default ({ getService }: FtrProviderContext) => { '@timestamp': laterTimestamp, host: { name: 'host-b' }, }; - + const doc2WithLaterTimestamp = { + ...doc1WithLaterTimestamp, + '@timestamp': laterTimestamp2, + host: { name: 'host-c' }, + }; + const doc3WithLaterTimestamp = { + ...doc2WithLaterTimestamp, + '@timestamp': laterTimestamp3, + host: { name: 'host-d' }, + }; await indexListOfSourceDocuments([ doc1, doc1WithLaterTimestamp, - { ...doc1, '@timestamp': timestamp1, host: { name: 'host-c' } }, - { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2, host: { name: 'host-d' } }, + doc2WithLaterTimestamp, + doc3WithLaterTimestamp, ]); const rule: EqlRuleCreateProps = { @@ -2378,17 +2417,26 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(9); + expect(previewAlerts.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(6); - expect(sequenceAlert.length).toEqual(3); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); - expect(sequenceAlert[0]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); - expect(sequenceAlert[1]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); - expect(sequenceAlert[2]?._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); }); it('suppresses alerts on a field with array values', async () => { @@ -2446,8 +2494,6 @@ export default ({ getService }: FtrProviderContext) => { ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_START]: timestamp2, - [ALERT_SUPPRESSION_END]: timestamp3, [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -2536,21 +2582,21 @@ export default ({ getService }: FtrProviderContext) => { (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - // for sequence alerts if neither of the fields are there, we cannot suppress? + // for sequence alerts if neither of the fields are there, we cannot suppress expect(sequenceAlert.length).toEqual(4); expect(sequenceAlert[0]._source).toEqual({ ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: ['agent-a'], + value: 'agent-a', }, { field: 'agent.version', - value: ['10'], + value: 10, }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); expect(sequenceAlert[1]._source).toEqual({ @@ -2562,10 +2608,10 @@ export default ({ getService }: FtrProviderContext) => { }, { field: 'agent.version', - value: ['10'], + value: 10, }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, }); expect(sequenceAlert[2]._source).toEqual({ @@ -2573,14 +2619,14 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: ['agent-a'], + value: null, }, { field: 'agent.version', value: null, }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, }); expect(sequenceAlert[3]._source).toEqual({ @@ -2588,7 +2634,7 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: null, + value: 'agent-a', }, { field: 'agent.version', @@ -2682,8 +2728,8 @@ export default ({ getService }: FtrProviderContext) => { (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - // for sequence alerts if neither of the fields are there, we cannot suppress? - expect(sequenceAlert.length).toEqual(9); + // for sequence alerts if neither of the fields are there, we cannot suppress + expect(sequenceAlert.length).toEqual(10); const [suppressedSequenceAlerts] = partition( sequenceAlert, (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] @@ -2694,14 +2740,14 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'agent.name', - value: ['agent-a'], + value: 'agent-a', }, { field: 'agent.version', - value: ['10'], + value: 10, }, ], - [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, }); }); @@ -2755,24 +2801,21 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: ['host.name', ALERT_ORIGINAL_TIME], }); - const [sequenceAlert] = partition( + const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - // for sequence alerts if neither of the fields are there, we cannot suppress? + expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); - expect(previewAlerts.length).toEqual(3); expect(sequenceAlert[0]._source).toEqual({ ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], - [ALERT_SUPPRESSION_START]: secondTimestamp, - [ALERT_SUPPRESSION_END]: secondTimestamp6, [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // if deduplication failed this would 12, or the previewAlerts count would be double }); }); @@ -2831,13 +2874,12 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], // suppression boundaries equal to original event time, since no alert been suppressed - [ALERT_SUPPRESSION_START]: firstTimestamp2, - [ALERT_SUPPRESSION_END]: firstTimestamp2, - [TIMESTAMP]: suppressionStart, + [ALERT_SUPPRESSION_START]: suppressionStart, + [ALERT_SUPPRESSION_END]: suppressionStart, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }) ); @@ -2879,11 +2921,11 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], - [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same - [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated + // [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same + // [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed }); // suppression end value should be greater than second document timestamp, but lesser than current time @@ -2947,13 +2989,13 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], // suppression boundaries equal to original event time, since no alert been suppressed - [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: firstTimestamp, - [TIMESTAMP]: suppressionStart, + // [ALERT_SUPPRESSION_START]: firstTimestamp, + // [ALERT_SUPPRESSION_END]: firstTimestamp, + // [TIMESTAMP]: suppressionStart, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }); @@ -2976,7 +3018,7 @@ export default ({ getService }: FtrProviderContext) => { { ...secondDocument, '@timestamp': secondTimestamp2 }, ]); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp = new Date(Date.now() + 1000); + const afterTimestamp = new Date(Date.now() + 5000); const secondAlerts = await getOpenAlerts( supertest, log, @@ -2998,11 +3040,11 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], - [ALERT_SUPPRESSION_START]: firstTimestamp, - [ALERT_SUPPRESSION_END]: firstTimestamp, + // [ALERT_SUPPRESSION_START]: firstTimestamp, + // [ALERT_SUPPRESSION_END]: firstTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed }); expect(sequenceAlert2[1]._source).toEqual({ @@ -3010,14 +3052,144 @@ export default ({ getService }: FtrProviderContext) => { [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', - value: ['host-a'], + value: 'host-a', }, ], - [ALERT_SUPPRESSION_START]: secondTimestamp2, - [ALERT_SUPPRESSION_END]: secondTimestamp2, + // [ALERT_SUPPRESSION_START]: secondTimestamp2, + // [ALERT_SUPPRESSION_END]: secondTimestamp2, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed }); }); + + it('does not suppress alerts outside of duration when query with 3 sequences', async () => { + const id = uuidv4(); + // this timestamp is 1 minute in the past + const dateNow = Date.now(); + const timestampSequenceEvent1 = new Date(dateNow - 5000).toISOString(); + const timestampSequenceEvent2 = new Date(dateNow - 5500).toISOString(); + const timestampSequenceEvent3 = new Date(dateNow - 5800).toISOString(); + + const firstSequenceEvent = { + id, + '@timestamp': timestampSequenceEvent1, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstSequenceEvent, + { ...firstSequenceEvent, '@timestamp': timestampSequenceEvent2 }, + { ...firstSequenceEvent, '@timestamp': timestampSequenceEvent3 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: `sequence [any where id == "${id}"] [any where id == "${id}"] [any where id == "${id}"]`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 7, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-10s', + interval: '5s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one shell alert + // and three building block alerts + expect(alerts.hits.hits.length).toEqual(4); + const [sequenceAlert, buildingBlockAlerts] = partition( + alerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(3); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + + const dateNow2 = Date.now(); + const secondTimestampEventSequence = new Date(dateNow2 - 2000).toISOString(); + const secondTimestampEventSequence2 = new Date(dateNow2 - 1000).toISOString(); + const secondTimestampEventSequence3 = new Date(dateNow2).toISOString(); + + const secondSequenceEvent = { + id, + '@timestamp': secondTimestampEventSequence, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. + // the second rule run should generate a new alert + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + + await indexListOfSourceDocuments([ + secondSequenceEvent, + { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence2 }, + { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, + ]); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp2 = new Date(dateNow2 + 2000); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp2 + ); + + const [sequenceAlert2, buildingBlockAlerts2] = partition( + secondAlerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + + // two sequence alerts because the second one happened + // outside of the rule's suppression duration + expect(sequenceAlert2.length).toEqual(2); + expect(buildingBlockAlerts2.length).toEqual(6); + // timestamps should be different for two alerts, showing they were + // created in different rule executions + expect(sequenceAlert2[0]?._source?.[TIMESTAMP]).not.toEqual( + sequenceAlert2[1]?._source?.[TIMESTAMP] + ); + + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // only one alert created, so zero documents are suppressed + }); + expect(sequenceAlert2[1]._source).toEqual({ + ...sequenceAlert2[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // only one alert created, so zero documents are suppressed + }); + }); }); }); }; From fda2a8404d06b1df721c3566b679281e7fa1e28b Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 4 Nov 2024 23:21:24 -0500 Subject: [PATCH 37/70] evaluate do not suppress for values from generated shell alert, not from individual events in the sequence --- .../eql/build_alert_group_from_sequence.ts | 5 ++ .../rule_types/eql/create_eql_alert_type.ts | 45 ++++++++++ .../detection_engine/rule_types/eql/eql.ts | 9 ++ ...bulk_create_suppressed_alerts_in_memory.ts | 83 +++++++++++------ .../utils/bulk_create_with_suppression.ts | 22 +++-- .../utils/partition_missing_fields_events.ts | 13 ++- .../rule_types/utils/utils.ts | 84 +++++++++++++++++ .../utils/wrap_suppressed_alerts.ts | 90 +++++++++---------- .../eql_alert_suppression.ts | 43 ++++----- 9 files changed, 285 insertions(+), 109 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 2a7f4c291fd0e..3232c7b9e6ba9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -72,6 +72,11 @@ export type BuildAlertGroupFromSequenceReturnType = [ ...Array> ]; +export type AlertGroupFromSequenceBuilder = ( + sequence: EqlSequence, + buildReasonMessage: BuildReasonMessage +) => BuildAlertGroupFromSequenceReturnType; + /** * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index ccf49cfb5e8b9..6533eda8c8fb5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -14,12 +14,19 @@ import { EqlRuleParams } from '../../rule_schema'; import { eqlExecutor } from './eql'; import type { CreateRuleOptions, SecurityAlertType, SignalSource, SignalSourceHit } from '../types'; import { validateIndexPatterns } from '../utils'; +import { sequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; import type { BuildReasonMessage } from '../utils/reason_formatters'; import { wrapSuppressedAlerts, wrapSuppressedSequenceAlerts, } from '../utils/wrap_suppressed_alerts'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; +import type { WrappedEqlShellOptionalSubAlertsType } from './build_alert_group_from_sequence'; +import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence'; +import type { + EqlBuildingBlockFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; export const createEqlAlertType = ( createOptions: CreateRuleOptions @@ -129,6 +136,42 @@ export const createEqlAlertType = ( alertSuppression: completeRule.ruleParams.alertSuppression, licensing, }); + const alertGroupFromSequenceBuilder = ( + sequence: EqlHitsSequence, + buildReasonMessage: BuildReasonMessage + ) => + buildAlertGroupFromSequence({ + ruleExecutionLogger, + sequence, + completeRule, + mergeStrategy, + spaceId, + buildReasonMessage, + indicesToQuery: inputIndex, + alertTimestampOverride, + applyOverrides: true, + publicBaseUrl, + }); + + const addSequenceSuppressionTermsAndFields = ( + shellAlert: WrappedEqlShellOptionalSubAlertsType, + buildingBlockAlerts: Array>, + buildReasonMessage: BuildReasonMessage + ) => + sequenceSuppressionTermsAndFieldsFactory({ + shellAlert, + buildingBlockAlerts, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + }); const { result, loggedRequests } = await eqlExecutor({ completeRule, tuple, @@ -146,6 +189,8 @@ export const createEqlAlertType = ( unprocessedExceptions, wrapSuppressedHits, wrapSuppressedSequences, + alertGroupFromSequenceBuilder, + addSequenceSuppressionTermsAndFields, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 1c89fa02540ec..138221b712c81 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -30,6 +30,7 @@ import type { WrapSuppressedSequences, CreateRuleOptions, } from '../types'; +import type { SequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -54,6 +55,7 @@ import { getDataTierFilter } from '../utils/get_data_tier_filter'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logEqlRequest } from '../utils/logged_requests'; import * as i18n from '../translations'; +import type { AlertGroupFromSequenceBuilder } from './build_alert_group_from_sequence'; interface EqlExecutorParams { inputIndex: string[]; @@ -72,6 +74,8 @@ interface EqlExecutorParams { unprocessedExceptions: ExceptionListItemSchema[]; wrapSuppressedHits: WrapSuppressedHits; wrapSuppressedSequences: WrapSuppressedSequences; + alertGroupFromSequenceBuilder: AlertGroupFromSequenceBuilder; + addSequenceSuppressionTermsAndFields: SequenceSuppressionTermsAndFieldsFactory; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; isAlertSuppressionActive: boolean; @@ -97,6 +101,8 @@ export const eqlExecutor = async ({ unprocessedExceptions, wrapSuppressedHits, wrapSuppressedSequences, + alertGroupFromSequenceBuilder, + addSequenceSuppressionTermsAndFields, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, @@ -203,9 +209,12 @@ export const eqlExecutor = async ({ tuple, alertSuppression: completeRule.ruleParams.alertSuppression, wrapSuppressedSequences, + alertGroupFromSequenceBuilder, + addSequenceSuppressionTermsAndFields, alertTimestampOverride, alertWithSuppression, experimentalFeatures, + mergeSourceAndFields: true, }); } else { newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index cf07780f0a599..4165dcd64266c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -19,6 +19,7 @@ import type { WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; +import type { SequenceSuppressionTermsAndFieldsFactory } from './utils'; import { addToSearchAfterReturn } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; @@ -30,8 +31,11 @@ import type { ExperimentalFeatures } from '../../../../../common'; import type { BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; +import type { AlertGroupFromSequenceBuilder } from '../eql/build_alert_group_from_sequence'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -82,6 +86,8 @@ export interface BulkCreateSuppressedSequencesParams experimentalFeatures: ExperimentalFeatures; mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; + alertGroupFromSequenceBuilder: AlertGroupFromSequenceBuilder; + addSequenceSuppressionTermsAndFields: SequenceSuppressionTermsAndFieldsFactory; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -160,6 +166,8 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ tuple, alertSuppression, wrapSuppressedSequences, + alertGroupFromSequenceBuilder, + addSequenceSuppressionTermsAndFields, alertWithSuppression, alertTimestampOverride, experimentalFeatures, @@ -170,41 +178,62 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === AlertSuppressionMissingFieldsStrategyEnum.suppress; - let suppressibleSequences: Array> = []; + const suppressibleWrappedSequences: Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } + > = []; const unsuppressibleWrappedDocs: Array> = []; - if (!suppressOnMissingFields) { - sequences.forEach((sequence) => { - // if none of the events in the given sequence - // contain a value, then wrap sequence normally, - // otherwise wrap as suppressed - // ask product - const [eventsWithFields] = partitionMissingFieldsEvents( - sequence.events, - alertSuppression?.groupBy || [], - ['fields'], - mergeSourceAndFields - ); + sequences.forEach((sequence) => { + const alertGroupFromSequence = alertGroupFromSequenceBuilder(sequence, buildReasonMessage); + const shellAlert = alertGroupFromSequence?.[0]; + if (shellAlert != null) { + if (!suppressOnMissingFields) { + const [shellAlertWithFields] = partitionMissingFieldsEvents( + shellAlert != null ? [shellAlert] : [], + alertSuppression?.groupBy || [], + ['fields'], + mergeSourceAndFields + ); - if (eventsWithFields.length === 0) { - // unsuppressible sequence alert - const wrappedSequence = wrapSequences([sequence], buildReasonMessage); - unsuppressibleWrappedDocs.push(...wrappedSequence); + if (shellAlertWithFields.length === 0) { + // unsuppressible sequence alert + // directly push alertGroup because these docs are wrapped already + unsuppressibleWrappedDocs.push( + ...(alertGroupFromSequence as Array< + | WrappedFieldsLatest + | WrappedFieldsLatest + >) + ); + } else { + const wrappedWithSuppressionTerms = addSequenceSuppressionTermsAndFields( + shellAlert, + alertGroupFromSequence.slice(1) as Array< + WrappedFieldsLatest + >, + buildReasonMessage + ); + suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); + } } else { - suppressibleSequences.push(sequence); + const wrappedWithSuppressionTerms = addSequenceSuppressionTermsAndFields( + shellAlert, + alertGroupFromSequence.slice(1) as Array< + WrappedFieldsLatest + >, + buildReasonMessage + ); + suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } - }); - } else { - suppressibleSequences = sequences; - } + } + }); - const suppressibleWrappedDocsWithSubAlerts = wrapSuppressedSequences( - suppressibleSequences, - buildReasonMessage - ); + console.error('suppressible Sequences', suppressibleWrappedSequences.length); + console.error('unsuppressibleWrappedDocs Sequences', unsuppressibleWrappedDocs.length); return executeBulkCreateAlerts({ - suppressibleWrappedDocs: suppressibleWrappedDocsWithSubAlerts, + suppressibleWrappedDocs: suppressibleWrappedSequences, unsuppressibleWrappedDocs, toReturn, bulkCreate, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index e88ff2ec0bf71..e9be216a2bf03 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -99,17 +99,21 @@ export const bulkCreateWithSuppression = async < } }; + const alerts = wrappedDocs.map((doc) => ({ + _id: doc._id, + // `fields` should have already been merged into `doc._source` + _source: doc._source, + subAlerts: + doc?.subAlerts != null + ? doc?.subAlerts?.map((subAlert) => ({ _id: subAlert._id, _source: subAlert._source })) + : undefined, + })); + + console.error('DO WE HAVE ALERTS TO SUPPRESS?', alerts.length); + const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( - wrappedDocs.map((doc) => ({ - _id: doc._id, - // `fields` should have already been merged into `doc._source` - _source: doc._source, - subAlerts: - doc?.subAlerts != null - ? doc?.subAlerts?.map((subAlert) => ({ _id: subAlert._id, _source: subAlert._source })) - : undefined, - })), + alerts, suppressionWindow, enrichAlertsWrapper, alertTimestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index 901768fe5c773..359448034f9d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -6,6 +6,7 @@ */ import pick from 'lodash/pick'; +import has from 'lodash/has'; import get from 'lodash/get'; import partition from 'lodash/partition'; @@ -22,21 +23,27 @@ export const partitionMissingFieldsEvents = < events: T[], suppressedBy: string[] = [], // path to fields property within event object. At this point, it can be in root of event object or within event key - fieldsPath: ['event', 'fields'] | ['fields'] | [] = [], + fieldsPath: ['event', 'fields'] | ['fields'] | ['_source'] | [] = [], mergeSourceAndFields: boolean = false ): T[][] => { return partition(events, (event) => { if (suppressedBy.length === 0) { return true; } + // console.error('SUPPRESSED BY', suppressedBy); const eventFields = fieldsPath.length ? get(event, fieldsPath) : event; + // console.error('EVENT FIELDS', eventFields); const sourceFields = (event as SignalSourceHit)?._source || (event as { event: SignalSourceHit })?.event?._source; const fields = mergeSourceAndFields ? { ...sourceFields, ...eventFields } : eventFields; + return suppressedBy.every((suppressionPath) => has(fields, suppressionPath)); + // const picked = pick(fields, suppressedBy); + // console.error('PICKED', picked); + // console.error('PICK FIELDS', Object.keys(picked)); - const hasMissingFields = Object.keys(pick(fields, suppressedBy)).length < suppressedBy.length; + // const hasMissingFields = Object.keys(picked).length < suppressedBy.length; - return !hasMissingFields; + // return !hasMissingFields; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index 160cb449bc030..fd45913ef2a0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -7,6 +7,7 @@ import { createHash } from 'crypto'; import { chunk, get, invert, isEmpty, partition } from 'lodash'; import moment from 'moment'; +import objectHash from 'object-hash'; import dateMath from '@kbn/datemath'; import { isCCSRemoteIndexName } from '@kbn/es-query'; @@ -56,6 +57,7 @@ import type { } from '../types'; import type { ShardError } from '../../../types'; import type { + CompleteRule, EqlRuleParams, EsqlRuleParams, MachineLearningRuleParams, @@ -70,6 +72,9 @@ import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { BaseFieldsLatest, DetectionAlert, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, + WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants'; import type { GenericBulkCreateResponse } from '../factories'; @@ -78,6 +83,14 @@ import type { EqlShellAlert800, } from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; +import { + ExtraFieldsForShellAlert, + WrappedEqlShellOptionalSubAlertsType, +} from '../eql/build_alert_group_from_sequence'; +import { ConfigType } from '@kbn/security-solution-plugin/server/config'; +import { BuildReasonMessage } from './reason_formatters'; +import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; +import { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; export const MAX_RULE_GAP_RATIO = 4; @@ -1046,3 +1059,74 @@ export const isEqlBuildingBlockAlert = ( export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAlert800 => (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; + +type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; + +export interface SequenceSuppressionTermsAndFieldsParams { + shellAlert: WrappedEqlShellOptionalSubAlertsType; + buildingBlockAlerts: Array>; + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + buildReasonMessage: BuildReasonMessage; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; +} + +export type SequenceSuppressionTermsAndFieldsFactory = ( + shellAlert: WrappedEqlShellOptionalSubAlertsType, + buildingBlockAlerts: Array>, + buildReasonMessage: BuildReasonMessage +) => WrappedFieldsLatest & { + subAlerts: Array>; +}; + +export const sequenceSuppressionTermsAndFieldsFactory = ({ + shellAlert, + buildingBlockAlerts, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, +}: SequenceSuppressionTermsAndFieldsParams): WrappedFieldsLatest< + EqlShellFieldsLatest & SuppressionFieldsLatest +> & { + subAlerts: Array>; +} => { + console.error('ARE WE IN HERE'); + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields: shellAlert?._source, + }); + console.error('SUPPRESSION TERMS', suppressionTerms); + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const suppressionFields = getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + // as casting should work because the alert fields are flattened (hopefully?) + fields: shellAlert?._source as Record | undefined, + suppressionTerms, + fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), + instanceId, + }); + console.error('SUPPRESSION FIELDS', JSON.stringify(suppressionFields, null, 2)); + const theFields = Object.keys(suppressionFields) as Array; + // mutates shell alert to contain values from suppression fields + theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); + shellAlert.subAlerts = buildingBlockAlerts; + + return shellAlert as WrappedFieldsLatest & { + subAlerts: Array>; + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 147e029fe92bd..92c2154959d7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -31,7 +31,6 @@ import { generateId, isEqlShellAlert } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; -import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; @@ -161,50 +160,51 @@ export const wrapSuppressedSequenceAlerts = ({ >, sequence ) => { - const alertGroupFromSequence = buildAlertGroupFromSequence({ - ruleExecutionLogger, - sequence, - completeRule, - mergeStrategy, - spaceId, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - applyOverrides: true, - publicBaseUrl, - }); - const shellAlert = alertGroupFromSequence[0]; - if (!isEqlShellAlert(shellAlert)) { - return [...acc]; - } - const buildingBlocks = alertGroupFromSequence.slice(1) as Array< - WrappedFieldsLatest - >; - const suppressionTerms = getSuppressionTerms({ - alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: shellAlert?._source, - }); - const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - - const suppressionFields = getSuppressionAlertFields({ - primaryTimestamp, - secondaryTimestamp, - // as casting should work because the alert fields are flattened (hopefully?) - fields: shellAlert?._source as Record | undefined, - suppressionTerms, - fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), - instanceId, - }); - const theFields = Object.keys(suppressionFields) as Array; - // mutates shell alert to contain values from suppression fields - theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); - shellAlert.subAlerts = buildingBlocks; - - return [...acc, shellAlert] as Array< - WrappedFieldsLatest & { - subAlerts: Array>; - } - >; + // const alertGroupFromSequence = buildAlertGroupFromSequence({ + // ruleExecutionLogger, + // sequence, + // completeRule, + // mergeStrategy, + // spaceId, + // buildReasonMessage, + // indicesToQuery, + // alertTimestampOverride, + // applyOverrides: true, + // publicBaseUrl, + // }); + // const shellAlert = alertGroupFromSequence[0]; + // // have to check for null to satisfy type assertions below + // // otherwise i would still have to write shellAlert?.propert + // if (shellAlert == null || !isEqlShellAlert(shellAlert?._source)) { + // return [...acc]; + // } + // const buildingBlocks = alertGroupFromSequence.slice(1) as Array< + // WrappedFieldsLatest + // >; + // const suppressionTerms = getSuppressionTerms({ + // alertSuppression: completeRule?.ruleParams?.alertSuppression, + // fields: shellAlert?._source, + // }); + // const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + // const suppressionFields = getSuppressionAlertFields({ + // primaryTimestamp, + // secondaryTimestamp, + // // as casting should work because the alert fields are flattened (hopefully?) + // fields: shellAlert?._source as Record | undefined, + // suppressionTerms, + // fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), + // instanceId, + // }); + // const theFields = Object.keys(suppressionFields) as Array; + // // mutates shell alert to contain values from suppression fields + // theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); + // shellAlert.subAlerts = buildingBlocks; + // return [...acc, shellAlert] as Array< + // WrappedFieldsLatest & { + // subAlerts: Array>; + // } + // >; + return acc; }, [] as Array< WrappedFieldsLatest & { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 811d5de0ea829..e0a83e4901544 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -1999,20 +1999,25 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, - sort: [ALERT_ORIGINAL_TIME], + sort: [TIMESTAMP], }); - // we expect one alert and two suppressed alerts + // we expect one alert and one suppressed alert // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(6); - const [sequenceAlert, buildingBlockAlerts] = partition( + const [sequenceAlerts, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); expect(buildingBlockAlerts.length).toEqual(4); - expect(sequenceAlert.length).toEqual(2); + expect(sequenceAlerts.length).toEqual(2); + const [suppressedSequenceAlerts] = partition( + sequenceAlerts, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 + ); + expect(suppressedSequenceAlerts.length).toEqual(1); - expect(sequenceAlert[0]?._source).toEqual({ - ...sequenceAlert[0]?._source, + expect(suppressedSequenceAlerts[0]._source).toEqual({ + ...suppressedSequenceAlerts[0]._source, [ALERT_SUPPRESSION_TERMS]: [ { field: 'host.name', @@ -2020,19 +2025,6 @@ export default ({ getService }: FtrProviderContext) => { }, ], [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', - [ALERT_SUPPRESSION_DOCS_COUNT]: 0, - }); - expect(sequenceAlert[1]?._source).toEqual({ - ...sequenceAlert[1]?._source, - [ALERT_SUPPRESSION_TERMS]: [ - { - field: 'host.name', - value: null, - }, - ], - [TIMESTAMP]: '2020-10-28T07:00:00.000Z', - [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }); }); @@ -2081,10 +2073,11 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId, - sort: [ALERT_ORIGINAL_TIME], + sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that + // console.error(JSON.stringify(previewAlerts, null, 2)); expect(previewAlerts.length).toEqual(6); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, @@ -2092,7 +2085,7 @@ export default ({ getService }: FtrProviderContext) => { ); const [suppressedSequenceAlerts] = partition( sequenceAlert, - (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 ); expect(buildingBlockAlerts.length).toEqual(4); expect(sequenceAlert.length).toEqual(2); @@ -2168,7 +2161,7 @@ export default ({ getService }: FtrProviderContext) => { ); const [suppressedSequenceAlerts] = partition( sequenceAlert, - (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 ); expect(suppressedSequenceAlerts.length).toEqual(1); expect(buildingBlockAlerts.length).toEqual(2); @@ -2257,7 +2250,7 @@ export default ({ getService }: FtrProviderContext) => { ); const [suppressedSequenceAlerts] = partition( sequenceAlert, - (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 ); // no alerts should be suppressed because doNotSuppress is set expect(suppressedSequenceAlerts.length).toEqual(0); @@ -2721,10 +2714,10 @@ export default ({ getService }: FtrProviderContext) => { ); // for sequence alerts if neither of the fields are there, we cannot suppress - expect(sequenceAlert.length).toEqual(10); + // expect(sequenceAlert.length).toEqual(10); const [suppressedSequenceAlerts] = partition( sequenceAlert, - (alert) => alert?._source?.['kibana.alert.suppression.docs_count'] + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 ); expect(suppressedSequenceAlerts.length).toEqual(1); expect(suppressedSequenceAlerts[0]._source).toEqual({ From ac0c179b60889dfa9bc9c96a25844e432979e391 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 5 Nov 2024 04:33:00 +0000 Subject: [PATCH 38/70] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/security_solution/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index f8fdcfcd8f438..a83667cfb77de 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -228,5 +228,6 @@ "@kbn/data-stream-adapter", "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", + "@kbn/security-solution-plugin", ] } From 351d2239b3f9e1f2a8d882d95d6e8bf100464895 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 5 Nov 2024 04:53:25 +0000 Subject: [PATCH 39/70] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../rule_types/utils/partition_missing_fields_events.ts | 1 - .../server/lib/detection_engine/rule_types/utils/utils.ts | 8 ++++---- .../rule_types/utils/wrap_suppressed_alerts.ts | 4 +--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index 359448034f9d7..512223cd7ef91 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -5,7 +5,6 @@ * 2.0. */ -import pick from 'lodash/pick'; import has from 'lodash/has'; import get from 'lodash/get'; import partition from 'lodash/partition'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index fd45913ef2a0f..5485d547f4f4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -39,6 +39,8 @@ import type { import { parseDuration } from '@kbn/alerting-plugin/server'; import type { ExceptionListClient, ListClient, ListPluginSetup } from '@kbn/lists-plugin/server'; import type { SanitizedRuleAction } from '@kbn/alerting-plugin/common'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { ConfigType } from '../../../../config'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; import type { Privilege } from '../../../../../common/api/detection_engine'; import { RuleExecutionStatusEnum } from '../../../../../common/api/detection_engine/rule_monitoring'; @@ -83,14 +85,12 @@ import type { EqlShellAlert800, } from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; -import { +import type { ExtraFieldsForShellAlert, WrappedEqlShellOptionalSubAlertsType, } from '../eql/build_alert_group_from_sequence'; -import { ConfigType } from '@kbn/security-solution-plugin/server/config'; -import { BuildReasonMessage } from './reason_formatters'; +import type { BuildReasonMessage } from './reason_formatters'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; export const MAX_RULE_GAP_RATIO = 4; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 92c2154959d7b..a9a26b8e9cc14 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -15,7 +15,6 @@ import type { SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, - EqlBuildingBlockFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; import type { @@ -27,10 +26,9 @@ import type { import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { generateId, isEqlShellAlert } from './utils'; +import { generateId } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; -import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; From af229f81d83b9b55fea2a93f2f702261fc7dbcde Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 5 Nov 2024 05:05:26 +0000 Subject: [PATCH 40/70] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/security_solution/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index a83667cfb77de..f8fdcfcd8f438 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -228,6 +228,5 @@ "@kbn/data-stream-adapter", "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", - "@kbn/security-solution-plugin", ] } From 0412e2593ae4c596625344ff688d7112cf747356 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 7 Nov 2024 08:57:36 -0500 Subject: [PATCH 41/70] adds eql utility class to encapsulate related params and centralize definitions for alert wrapping and other supplemental functions --- .../rule_types/eql/create_eql_alert_type.ts | 103 ++---------- .../rule_types/eql/eql.test.ts | 14 +- .../detection_engine/rule_types/eql/eql.ts | 23 +-- .../rule_types/eql/eql_utils.ts | 154 ++++++++++++++++++ ...bulk_create_suppressed_alerts_in_memory.ts | 40 ++--- .../utils/partition_missing_fields_events.ts | 14 +- .../rule_types/utils/utils.ts | 21 +-- .../utils/wrap_suppressed_alerts.ts | 5 +- 8 files changed, 207 insertions(+), 167 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 6533eda8c8fb5..b8c672c88af44 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -7,26 +7,14 @@ import { EQL_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; -import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; import { SERVER_APP_ID } from '../../../../../common/constants'; import { EqlRuleParams } from '../../rule_schema'; import { eqlExecutor } from './eql'; -import type { CreateRuleOptions, SecurityAlertType, SignalSource, SignalSourceHit } from '../types'; +import type { CreateRuleOptions, SecurityAlertType } from '../types'; import { validateIndexPatterns } from '../utils'; -import { sequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; -import type { BuildReasonMessage } from '../utils/reason_formatters'; -import { - wrapSuppressedAlerts, - wrapSuppressedSequenceAlerts, -} from '../utils/wrap_suppressed_alerts'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; -import type { WrappedEqlShellOptionalSubAlertsType } from './build_alert_group_from_sequence'; -import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence'; -import type { - EqlBuildingBlockFieldsLatest, - WrappedFieldsLatest, -} from '../../../../../common/api/detection_engine/model/alerts'; +import { EqlUtils } from './eql_utils'; export const createEqlAlertType = ( createOptions: CreateRuleOptions @@ -97,81 +85,23 @@ export const createEqlAlertType = ( spaceId, } = execOptions; - const wrapSuppressedHits = ( - events: SignalSourceHit[], - buildReasonMessage: BuildReasonMessage - ) => - wrapSuppressedAlerts({ - events, - spaceId, - completeRule, - mergeStrategy, - indicesToQuery: inputIndex, - buildReasonMessage, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - primaryTimestamp, - secondaryTimestamp, - intendedTimestamp, - }); - const wrapSuppressedSequences = ( - sequences: Array>, - buildReasonMessage: BuildReasonMessage - ) => - wrapSuppressedSequenceAlerts({ - sequences, - spaceId, - completeRule, - mergeStrategy, - indicesToQuery: inputIndex, - buildReasonMessage, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - primaryTimestamp, - secondaryTimestamp, - }); const isAlertSuppressionActive = await getIsAlertSuppressionActive({ alertSuppression: completeRule.ruleParams.alertSuppression, licensing, }); - const alertGroupFromSequenceBuilder = ( - sequence: EqlHitsSequence, - buildReasonMessage: BuildReasonMessage - ) => - buildAlertGroupFromSequence({ - ruleExecutionLogger, - sequence, - completeRule, - mergeStrategy, - spaceId, - buildReasonMessage, - indicesToQuery: inputIndex, - alertTimestampOverride, - applyOverrides: true, - publicBaseUrl, - }); - const addSequenceSuppressionTermsAndFields = ( - shellAlert: WrappedEqlShellOptionalSubAlertsType, - buildingBlockAlerts: Array>, - buildReasonMessage: BuildReasonMessage - ) => - sequenceSuppressionTermsAndFieldsFactory({ - shellAlert, - buildingBlockAlerts, - spaceId, - completeRule, - mergeStrategy, - indicesToQuery: inputIndex, - buildReasonMessage, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - primaryTimestamp, - secondaryTimestamp, - }); + const eqlUtils = new EqlUtils({ + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + intendedTimestamp, + }); const { result, loggedRequests } = await eqlExecutor({ completeRule, tuple, @@ -187,10 +117,7 @@ export const createEqlAlertType = ( secondaryTimestamp, exceptionFilter, unprocessedExceptions, - wrapSuppressedHits, - wrapSuppressedSequences, - alertGroupFromSequenceBuilder, - addSequenceSuppressionTermsAndFields, + eqlUtils, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index cdcd0145fd1ff..3df3d9a291330 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -18,6 +18,7 @@ import { getCompleteRuleMock, getEqlRuleParams } from '../../rule_schema/mocks'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; import { eqlExecutor } from './eql'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; +import type { IEqlUtils } from './eql_utils'; jest.mock('../../routes/index/get_index_version'); jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() })); @@ -38,6 +39,7 @@ describe('eql_executor', () => { }; const mockExperimentalFeatures = {} as ExperimentalFeatures; const mockScheduleNotificationResponseActionsService = jest.fn(); + const mockEqlUtils = {} as IEqlUtils; beforeEach(() => { jest.clearAllMocks(); @@ -65,12 +67,11 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), - wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [getExceptionListItemSchemaMock()], - wrapSuppressedHits: jest.fn(), alertTimestampOverride: undefined, + eqlUtils: mockEqlUtils, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, experimentalFeatures: mockExperimentalFeatures, @@ -102,11 +103,10 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), - wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - wrapSuppressedHits: jest.fn(), + eqlUtils: mockEqlUtils, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, @@ -131,8 +131,7 @@ describe('eql_executor', () => { primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - wrapSuppressedHits: jest.fn(), - wrapSuppressedSequences: jest.fn(), + eqlUtils: mockEqlUtils, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, @@ -169,11 +168,10 @@ describe('eql_executor', () => { bulkCreate: jest.fn(), wrapHits: jest.fn(), wrapSequences: jest.fn(), - wrapSuppressedSequences: jest.fn(), primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - wrapSuppressedHits: jest.fn(), + eqlUtils: mockEqlUtils, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 138221b712c81..1e22d74e3519d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -26,11 +26,8 @@ import type { RuleRangeTuple, SearchAfterAndBulkCreateReturnType, SignalSource, - WrapSuppressedHits, - WrapSuppressedSequences, CreateRuleOptions, } from '../types'; -import type { SequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -55,7 +52,7 @@ import { getDataTierFilter } from '../utils/get_data_tier_filter'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logEqlRequest } from '../utils/logged_requests'; import * as i18n from '../translations'; -import type { AlertGroupFromSequenceBuilder } from './build_alert_group_from_sequence'; +import type { IEqlUtils } from './eql_utils'; interface EqlExecutorParams { inputIndex: string[]; @@ -67,15 +64,12 @@ interface EqlExecutorParams { version: string; bulkCreate: BulkCreate; wrapHits: WrapHits; + eqlUtils: IEqlUtils; wrapSequences: WrapSequences; primaryTimestamp: string; secondaryTimestamp?: string; exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; - wrapSuppressedHits: WrapSuppressedHits; - wrapSuppressedSequences: WrapSuppressedSequences; - alertGroupFromSequenceBuilder: AlertGroupFromSequenceBuilder; - addSequenceSuppressionTermsAndFields: SequenceSuppressionTermsAndFieldsFactory; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; isAlertSuppressionActive: boolean; @@ -99,10 +93,7 @@ export const eqlExecutor = async ({ secondaryTimestamp, exceptionFilter, unprocessedExceptions, - wrapSuppressedHits, - wrapSuppressedSequences, - alertGroupFromSequenceBuilder, - addSequenceSuppressionTermsAndFields, + eqlUtils, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, @@ -185,7 +176,7 @@ export const eqlExecutor = async ({ ruleExecutionLogger, tuple, alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedHits, + wrapSuppressedHits: eqlUtils.wrapSuppressedHits, alertTimestampOverride, alertWithSuppression, experimentalFeatures, @@ -201,20 +192,16 @@ export const eqlExecutor = async ({ await bulkCreateSuppressedSequencesInMemory({ sequences, toReturn: result, - wrapSequences, bulkCreate, services, buildReasonMessage: buildReasonMessageForEqlAlert, ruleExecutionLogger, tuple, alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedSequences, - alertGroupFromSequenceBuilder, - addSequenceSuppressionTermsAndFields, + eqlUtils, alertTimestampOverride, alertWithSuppression, experimentalFeatures, - mergeSourceAndFields: true, }); } else { newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts new file mode 100644 index 0000000000000..770f33e34f52a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { + BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { BuildReasonMessage } from '../utils/reason_formatters'; +import type { RuleWithInMemorySuppression } from '../utils/utils'; +import { sequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; +import type { + BuildAlertGroupFromSequenceReturnType, + WrappedEqlShellOptionalSubAlertsType, +} from './build_alert_group_from_sequence'; +import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence'; +import type { SignalSource, SignalSourceHit } from '../types'; +import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; +import type { CompleteRule } from '../../rule_schema'; +import type { ConfigType } from '../../../../config'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; + +interface ConstructorParams { + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; + intendedTimestamp?: Date; +} + +/** + * Interface for EqlUtils + */ +export interface IEqlUtils { + wrapSuppressedHits( + events: SignalSourceHit[], + buildReasonMessage: BuildReasonMessage + ): Array>; + alertGroupFromSequenceBuilder( + sequence: EqlHitsSequence, + buildReasonMessage: BuildReasonMessage + ): BuildAlertGroupFromSequenceReturnType; + addSequenceSuppressionTermsAndFields( + shellAlert: WrappedEqlShellOptionalSubAlertsType, + buildingBlockAlerts: Array>, + buildReasonMessage: BuildReasonMessage + ): WrappedFieldsLatest & { + subAlerts: Array>; + }; +} + +export class EqlUtils implements IEqlUtils { + #spaceId: string; + #completeRule: CompleteRule; + #mergeStrategy: ConfigType['alertMergeStrategy']; + #indicesToQuery: string[]; + #alertTimestampOverride: Date | undefined; + #ruleExecutionLogger: IRuleExecutionLogForExecutors; + #publicBaseUrl: string | undefined; + #primaryTimestamp: string; + #secondaryTimestamp: string | undefined; + #intendedTimestamp: Date | undefined; + constructor({ + spaceId, + completeRule, + mergeStrategy, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + intendedTimestamp, + }: ConstructorParams) { + this.#spaceId = spaceId; + this.#completeRule = completeRule; + this.#mergeStrategy = mergeStrategy; + this.#indicesToQuery = indicesToQuery; + this.#alertTimestampOverride = alertTimestampOverride; + this.#ruleExecutionLogger = ruleExecutionLogger; + this.#publicBaseUrl = publicBaseUrl; + this.#primaryTimestamp = primaryTimestamp; + this.#secondaryTimestamp = secondaryTimestamp; + this.#intendedTimestamp = intendedTimestamp; + } + + public wrapSuppressedHits(events: SignalSourceHit[], buildReasonMessage: BuildReasonMessage) { + return wrapSuppressedAlerts({ + events, + buildReasonMessage, + spaceId: this.#spaceId, + completeRule: this.#completeRule, + mergeStrategy: this.#mergeStrategy, + indicesToQuery: this.#indicesToQuery, + alertTimestampOverride: this.#alertTimestampOverride, + ruleExecutionLogger: this.#ruleExecutionLogger, + publicBaseUrl: this.#publicBaseUrl, + primaryTimestamp: this.#primaryTimestamp, + secondaryTimestamp: this.#secondaryTimestamp, + intendedTimestamp: this.#intendedTimestamp, + }); + } + + public alertGroupFromSequenceBuilder( + sequence: EqlHitsSequence, + buildReasonMessage: BuildReasonMessage + ) { + return buildAlertGroupFromSequence({ + sequence, + buildReasonMessage, + applyOverrides: true, + spaceId: this.#spaceId, + completeRule: this.#completeRule, + mergeStrategy: this.#mergeStrategy, + indicesToQuery: this.#indicesToQuery, + alertTimestampOverride: this.#alertTimestampOverride, + ruleExecutionLogger: this.#ruleExecutionLogger, + publicBaseUrl: this.#publicBaseUrl, + }); + } + + public addSequenceSuppressionTermsAndFields( + shellAlert: WrappedEqlShellOptionalSubAlertsType, + buildingBlockAlerts: Array>, + buildReasonMessage: BuildReasonMessage + ) { + return sequenceSuppressionTermsAndFieldsFactory({ + shellAlert, + buildingBlockAlerts, + spaceId: this.#spaceId, + completeRule: this.#completeRule, + mergeStrategy: this.#mergeStrategy, + indicesToQuery: this.#indicesToQuery, + buildReasonMessage, + alertTimestampOverride: this.#alertTimestampOverride, + ruleExecutionLogger: this.#ruleExecutionLogger, + publicBaseUrl: this.#publicBaseUrl, + primaryTimestamp: this.#primaryTimestamp, + secondaryTimestamp: this.#secondaryTimestamp, + }); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 4165dcd64266c..554b129ba483c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -15,11 +15,9 @@ import type { WrapSuppressedHits, SignalSourceHit, SignalSource, - WrapSequences, WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; -import type { SequenceSuppressionTermsAndFieldsFactory } from './utils'; import { addToSearchAfterReturn } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; @@ -35,7 +33,8 @@ import type { EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; -import type { AlertGroupFromSequenceBuilder } from '../eql/build_alert_group_from_sequence'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; +import type { IEqlUtils } from '../eql/eql_utils'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -75,19 +74,15 @@ export interface BulkCreateSuppressedSequencesParams | 'ruleExecutionLogger' | 'tuple' | 'alertSuppression' - | 'wrapSuppressedSequences' | 'alertWithSuppression' | 'alertTimestampOverride' > { - wrapSequences: WrapSequences; sequences: Array>; buildingBlockAlerts?: Array>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; - mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; - alertGroupFromSequenceBuilder: AlertGroupFromSequenceBuilder; - addSequenceSuppressionTermsAndFields: SequenceSuppressionTermsAndFieldsFactory; + eqlUtils: IEqlUtils; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -158,20 +153,16 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ export const bulkCreateSuppressedSequencesInMemory = async ({ sequences, toReturn, - wrapSequences, bulkCreate, services, buildReasonMessage, ruleExecutionLogger, tuple, alertSuppression, - wrapSuppressedSequences, - alertGroupFromSequenceBuilder, - addSequenceSuppressionTermsAndFields, + eqlUtils, alertWithSuppression, alertTimestampOverride, experimentalFeatures, - mergeSourceAndFields = false, maxNumberOfAlertsMultiplier, }: BulkCreateSuppressedSequencesParams) => { const suppressOnMissingFields = @@ -186,18 +177,20 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ const unsuppressibleWrappedDocs: Array> = []; sequences.forEach((sequence) => { - const alertGroupFromSequence = alertGroupFromSequenceBuilder(sequence, buildReasonMessage); + const alertGroupFromSequence = eqlUtils.alertGroupFromSequenceBuilder( + sequence, + buildReasonMessage + ); const shellAlert = alertGroupFromSequence?.[0]; if (shellAlert != null) { if (!suppressOnMissingFields) { - const [shellAlertWithFields] = partitionMissingFieldsEvents( - shellAlert != null ? [shellAlert] : [], - alertSuppression?.groupBy || [], - ['fields'], - mergeSourceAndFields + // does the shell alert have all the suppression fields? + const hasEverySuppressionField = (alertSuppression?.groupBy || []).every( + (suppressionPath) => + robustGet({ key: suppressionPath, document: shellAlert._source }) != null ); - if (shellAlertWithFields.length === 0) { + if (!hasEverySuppressionField) { // unsuppressible sequence alert // directly push alertGroup because these docs are wrapped already unsuppressibleWrappedDocs.push( @@ -207,7 +200,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ >) ); } else { - const wrappedWithSuppressionTerms = addSequenceSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = eqlUtils.addSequenceSuppressionTermsAndFields( shellAlert, alertGroupFromSequence.slice(1) as Array< WrappedFieldsLatest @@ -217,7 +210,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } } else { - const wrappedWithSuppressionTerms = addSequenceSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = eqlUtils.addSequenceSuppressionTermsAndFields( shellAlert, alertGroupFromSequence.slice(1) as Array< WrappedFieldsLatest @@ -229,9 +222,6 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ } }); - console.error('suppressible Sequences', suppressibleWrappedSequences.length); - console.error('unsuppressibleWrappedDocs Sequences', unsuppressibleWrappedDocs.length); - return executeBulkCreateAlerts({ suppressibleWrappedDocs: suppressibleWrappedSequences, unsuppressibleWrappedDocs, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index 359448034f9d7..ef6b2c96cbb87 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -6,7 +6,6 @@ */ import pick from 'lodash/pick'; -import has from 'lodash/has'; import get from 'lodash/get'; import partition from 'lodash/partition'; @@ -23,27 +22,20 @@ export const partitionMissingFieldsEvents = < events: T[], suppressedBy: string[] = [], // path to fields property within event object. At this point, it can be in root of event object or within event key - fieldsPath: ['event', 'fields'] | ['fields'] | ['_source'] | [] = [], + fieldsPath: ['event', 'fields'] | ['fields'] | [] = [], mergeSourceAndFields: boolean = false ): T[][] => { return partition(events, (event) => { if (suppressedBy.length === 0) { return true; } - // console.error('SUPPRESSED BY', suppressedBy); const eventFields = fieldsPath.length ? get(event, fieldsPath) : event; - // console.error('EVENT FIELDS', eventFields); const sourceFields = (event as SignalSourceHit)?._source || (event as { event: SignalSourceHit })?.event?._source; const fields = mergeSourceAndFields ? { ...sourceFields, ...eventFields } : eventFields; - return suppressedBy.every((suppressionPath) => has(fields, suppressionPath)); - // const picked = pick(fields, suppressedBy); - // console.error('PICKED', picked); - // console.error('PICK FIELDS', Object.keys(picked)); + const hasMissingFields = Object.keys(pick(fields, suppressedBy)).length < suppressedBy.length; - // const hasMissingFields = Object.keys(picked).length < suppressedBy.length; - - // return !hasMissingFields; + return !hasMissingFields; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index fd45913ef2a0f..c925e126037bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -39,6 +39,7 @@ import type { import { parseDuration } from '@kbn/alerting-plugin/server'; import type { ExceptionListClient, ListClient, ListPluginSetup } from '@kbn/lists-plugin/server'; import type { SanitizedRuleAction } from '@kbn/alerting-plugin/common'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; import type { Privilege } from '../../../../../common/api/detection_engine'; import { RuleExecutionStatusEnum } from '../../../../../common/api/detection_engine/rule_monitoring'; @@ -83,14 +84,13 @@ import type { EqlShellAlert800, } from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; -import { +import type { ExtraFieldsForShellAlert, WrappedEqlShellOptionalSubAlertsType, } from '../eql/build_alert_group_from_sequence'; -import { ConfigType } from '@kbn/security-solution-plugin/server/config'; -import { BuildReasonMessage } from './reason_formatters'; +import type { ConfigType } from '../../../../config'; +import type { BuildReasonMessage } from './reason_formatters'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; export const MAX_RULE_GAP_RATIO = 4; @@ -1060,7 +1060,10 @@ export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAl (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; -type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; +export type RuleWithInMemorySuppression = + | ThreatRuleParams + | EqlRuleParams + | MachineLearningRuleParams; export interface SequenceSuppressionTermsAndFieldsParams { shellAlert: WrappedEqlShellOptionalSubAlertsType; @@ -1090,12 +1093,7 @@ export const sequenceSuppressionTermsAndFieldsFactory = ({ buildingBlockAlerts, spaceId, completeRule, - mergeStrategy, - indicesToQuery, - buildReasonMessage, alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, primaryTimestamp, secondaryTimestamp, }: SequenceSuppressionTermsAndFieldsParams): WrappedFieldsLatest< @@ -1103,12 +1101,10 @@ export const sequenceSuppressionTermsAndFieldsFactory = ({ > & { subAlerts: Array>; } => { - console.error('ARE WE IN HERE'); const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, fields: shellAlert?._source, }); - console.error('SUPPRESSION TERMS', suppressionTerms); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); const suppressionFields = getSuppressionAlertFields({ @@ -1120,7 +1116,6 @@ export const sequenceSuppressionTermsAndFieldsFactory = ({ fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), instanceId, }); - console.error('SUPPRESSION FIELDS', JSON.stringify(suppressionFields, null, 2)); const theFields = Object.keys(suppressionFields) as Array; // mutates shell alert to contain values from suppression fields theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 92c2154959d7b..feea0235ccb8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -15,7 +15,6 @@ import type { SignalSource, SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, - EqlBuildingBlockFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { ConfigType } from '../../../../config'; import type { @@ -27,10 +26,8 @@ import type { import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; -import { generateId, isEqlShellAlert } from './utils'; - +import { generateId } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; -import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; From 4e3587e0ed0e96ca14ebc2b993e198e853c76817 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 7 Nov 2024 10:00:39 -0500 Subject: [PATCH 42/70] flatten subAlerts and shell alerts when they are newAlerts --- .../create_persistence_rule_type_wrapper.ts | 26 ++++++------------- .../eql_alert_suppression.ts | 2 +- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index e820dcc2369e5..fcc38a5a6bc4a 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -547,7 +547,13 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ]; }); - let enrichedAlerts = newAlerts; + // we can now augment and enrich + // the sub alerts (if any) the same as we would + // any other newAlert + let enrichedAlerts = newAlerts.reduce((acc, newAlert) => { + const { subAlerts, ...everything } = newAlert; + return [...acc, everything, ...(subAlerts ?? [])]; + }, [] as typeof newAlerts); if (enrichAlerts) { try { @@ -571,24 +577,8 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper currentTimeOverride, }); - const augmentedBuildingBlockAlerts = - newAlerts != null && newAlerts.length > 0 - ? await augmentAlerts({ - alerts: newAlerts.flatMap((alert) => - alert.subAlerts != null ? alert.subAlerts : [] - ), - options, - kibanaVersion: ruleDataClient.kibanaVersion, - currentTimeOverride, - }) - : []; - const bulkResponse = await ruleDataClientWriter.bulk({ - body: [ - ...duplicateAlertUpdates, - ...mapAlertsToBulkCreate(augmentedAlerts), - ...mapAlertsToBulkCreate(augmentedBuildingBlockAlerts), - ], + body: [...duplicateAlertUpdates, ...mapAlertsToBulkCreate(augmentedAlerts)], refresh: true, }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index e0a83e4901544..79bbd5a6db2b6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -3128,7 +3128,7 @@ export default ({ getService }: FtrProviderContext) => { { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, ]); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp2 = new Date(dateNow2 + 2000); + const afterTimestamp2 = new Date(dateNow2 + 10000); const secondAlerts = await getOpenAlerts( supertest, log, From 3079b9eabcd2274281f1678704f1b09c5cc0e655 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 7 Nov 2024 15:16:51 -0500 Subject: [PATCH 43/70] fixes as castings --- .../eql/build_alert_group_from_sequence.ts | 9 ++- .../rule_types/eql/eql_utils.ts | 18 ++---- ...bulk_create_suppressed_alerts_in_memory.ts | 30 ++++----- .../rule_types/utils/suppression_utils.ts | 2 +- .../rule_types/utils/utils.ts | 64 ++++++++++++------- .../eql_alert_suppression.ts | 6 +- 6 files changed, 66 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 3232c7b9e6ba9..0541d4ba4c640 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -95,7 +95,9 @@ export const buildAlertGroupFromSequence = ({ alertTimestampOverride, publicBaseUrl, intendedTimestamp, -}: BuildAlertGroupFromSequence): BuildAlertGroupFromSequenceReturnType => { +}: BuildAlertGroupFromSequence): Array< + WrappedFieldsLatest +> => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { return []; @@ -188,10 +190,7 @@ export const buildAlertGroupFromSequence = ({ ); // sequence alert guaranteed to be first - return [sequenceAlert, ...wrappedBuildingBlocks] as [ - WrappedFieldsLatest?, - ...Array> - ]; + return [sequenceAlert, ...wrappedBuildingBlocks]; }; export interface BuildAlertRootParams { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts index 770f33e34f52a..b064bcf309505 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts @@ -15,7 +15,7 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import type { BuildReasonMessage } from '../utils/reason_formatters'; import type { RuleWithInMemorySuppression } from '../utils/utils'; -import { sequenceSuppressionTermsAndFieldsFactory } from '../utils/utils'; +import { buildShellAlertSuppressionTermsAndFields } from '../utils/utils'; import type { BuildAlertGroupFromSequenceReturnType, WrappedEqlShellOptionalSubAlertsType, @@ -52,10 +52,9 @@ export interface IEqlUtils { sequence: EqlHitsSequence, buildReasonMessage: BuildReasonMessage ): BuildAlertGroupFromSequenceReturnType; - addSequenceSuppressionTermsAndFields( + getShellAlertWithSuppressionTermsAndFields( shellAlert: WrappedEqlShellOptionalSubAlertsType, - buildingBlockAlerts: Array>, - buildReasonMessage: BuildReasonMessage + buildingBlockAlerts: Array> ): WrappedFieldsLatest & { subAlerts: Array>; }; @@ -131,22 +130,17 @@ export class EqlUtils implements IEqlUtils { }); } - public addSequenceSuppressionTermsAndFields( + public getShellAlertWithSuppressionTermsAndFields( shellAlert: WrappedEqlShellOptionalSubAlertsType, - buildingBlockAlerts: Array>, - buildReasonMessage: BuildReasonMessage + buildingBlockAlerts: Array> ) { - return sequenceSuppressionTermsAndFieldsFactory({ + return buildShellAlertSuppressionTermsAndFields({ shellAlert, buildingBlockAlerts, spaceId: this.#spaceId, completeRule: this.#completeRule, - mergeStrategy: this.#mergeStrategy, indicesToQuery: this.#indicesToQuery, - buildReasonMessage, alertTimestampOverride: this.#alertTimestampOverride, - ruleExecutionLogger: this.#ruleExecutionLogger, - publicBaseUrl: this.#publicBaseUrl, primaryTimestamp: this.#primaryTimestamp, secondaryTimestamp: this.#secondaryTimestamp, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 554b129ba483c..4071da60279ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -182,6 +182,12 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ buildReasonMessage ); const shellAlert = alertGroupFromSequence?.[0]; + const buildingBlocks = alertGroupFromSequence + .slice(1) + .filter( + (alertGroup): alertGroup is WrappedFieldsLatest => + alertGroup != null + ); if (shellAlert != null) { if (!suppressOnMissingFields) { // does the shell alert have all the suppression fields? @@ -189,33 +195,19 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ (suppressionPath) => robustGet({ key: suppressionPath, document: shellAlert._source }) != null ); - if (!hasEverySuppressionField) { - // unsuppressible sequence alert - // directly push alertGroup because these docs are wrapped already - unsuppressibleWrappedDocs.push( - ...(alertGroupFromSequence as Array< - | WrappedFieldsLatest - | WrappedFieldsLatest - >) - ); + unsuppressibleWrappedDocs.push(...[shellAlert, ...buildingBlocks]); } else { - const wrappedWithSuppressionTerms = eqlUtils.addSequenceSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = eqlUtils.getShellAlertWithSuppressionTermsAndFields( shellAlert, - alertGroupFromSequence.slice(1) as Array< - WrappedFieldsLatest - >, - buildReasonMessage + buildingBlocks ); suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } } else { - const wrappedWithSuppressionTerms = eqlUtils.addSequenceSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = eqlUtils.getShellAlertWithSuppressionTermsAndFields( shellAlert, - alertGroupFromSequence.slice(1) as Array< - WrappedFieldsLatest - >, - buildReasonMessage + buildingBlocks ); suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 552bac024b406..f63ca89384920 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -70,11 +70,11 @@ export const getSuppressionTerms = ({ alertSuppression: AlertSuppressionCamel | undefined; }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - const suppressedProps = pick(fields, suppressedBy) as Record< string, string[] | number[] | undefined >; + const suppressionTerms = suppressedBy.map((field) => { const value = get(suppressedProps, field) ?? null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index c925e126037bb..3a76559b75116 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -18,6 +18,12 @@ import { ALERT_RULE_UUID, ALERT_RULE_PARAMETERS, ALERT_BUILDING_BLOCK_TYPE, + TIMESTAMP, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_TERMS, } from '@kbn/rule-data-utils'; import type { ListArray, @@ -88,9 +94,9 @@ import type { ExtraFieldsForShellAlert, WrappedEqlShellOptionalSubAlertsType, } from '../eql/build_alert_group_from_sequence'; -import type { ConfigType } from '../../../../config'; import type { BuildReasonMessage } from './reason_formatters'; -import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; +import { getSuppressionTerms } from './suppression_utils'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; export const MAX_RULE_GAP_RATIO = 4; @@ -1066,16 +1072,12 @@ export type RuleWithInMemorySuppression = | MachineLearningRuleParams; export interface SequenceSuppressionTermsAndFieldsParams { - shellAlert: WrappedEqlShellOptionalSubAlertsType; + shellAlert: WrappedFieldsLatest; buildingBlockAlerts: Array>; spaceId: string; completeRule: CompleteRule; - mergeStrategy: ConfigType['alertMergeStrategy']; indicesToQuery: string[]; - buildReasonMessage: BuildReasonMessage; alertTimestampOverride: Date | undefined; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - publicBaseUrl: string | undefined; primaryTimestamp: string; secondaryTimestamp?: string; } @@ -1088,7 +1090,7 @@ export type SequenceSuppressionTermsAndFieldsFactory = ( subAlerts: Array>; }; -export const sequenceSuppressionTermsAndFieldsFactory = ({ +export const buildShellAlertSuppressionTermsAndFields = ({ shellAlert, buildingBlockAlerts, spaceId, @@ -1107,21 +1109,37 @@ export const sequenceSuppressionTermsAndFieldsFactory = ({ }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const suppressionFields = getSuppressionAlertFields({ - primaryTimestamp, - secondaryTimestamp, - // as casting should work because the alert fields are flattened (hopefully?) - fields: shellAlert?._source as Record | undefined, - suppressionTerms, - fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), - instanceId, - }); - const theFields = Object.keys(suppressionFields) as Array; - // mutates shell alert to contain values from suppression fields - theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); - shellAlert.subAlerts = buildingBlockAlerts; + const primarySuppressionTime = robustGet({ + key: primaryTimestamp, + document: shellAlert._source, + }) as string | undefined; + + const secondarySuppressionTime = + secondaryTimestamp && + (robustGet({ + key: secondaryTimestamp, + document: shellAlert._source, + }) as string | undefined); + + const suppressionTime = new Date( + primarySuppressionTime ?? + secondarySuppressionTime ?? + alertTimestampOverride ?? + shellAlert._source[TIMESTAMP] + ); - return shellAlert as WrappedFieldsLatest & { - subAlerts: Array>; + const suppressionFields: ExtraFieldsForShellAlert = { + [ALERT_INSTANCE_ID]: instanceId, + [ALERT_SUPPRESSION_TERMS]: suppressionTerms, + [ALERT_SUPPRESSION_START]: suppressionTime, + [ALERT_SUPPRESSION_END]: suppressionTime, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }; + + return { + _id: shellAlert._id, + _index: shellAlert._index, + _source: { ...shellAlert._source, ...suppressionFields }, + subAlerts: buildingBlockAlerts, }; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 79bbd5a6db2b6..6c1af2da52af2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -2002,8 +2002,8 @@ export default ({ getService }: FtrProviderContext) => { sort: [TIMESTAMP], }); // we expect one alert and one suppressed alert - // and two building block alerts, let's confirm that - expect(previewAlerts.length).toEqual(6); + // and two building block alerts per shell alert, let's confirm that + // expect(previewAlerts.length).toEqual(6); const [sequenceAlerts, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null @@ -3128,7 +3128,7 @@ export default ({ getService }: FtrProviderContext) => { { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, ]); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp2 = new Date(dateNow2 + 10000); + const afterTimestamp2 = new Date(dateNow2 + 15000); const secondAlerts = await getOpenAlerts( supertest, log, From 550ea89cf900297847c837766dbdb2d1031edc3d Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 7 Nov 2024 15:30:26 -0500 Subject: [PATCH 44/70] remove unused function wrapSuppressedSequences --- .../utils/wrap_suppressed_alerts.ts | 100 +----------------- 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index feea0235ccb8f..f11496bc140dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -9,9 +9,8 @@ import objectHash from 'object-hash'; import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import type { SignalSource, SignalSourceHit } from '../types'; +import type { SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, @@ -113,100 +112,3 @@ export const wrapSuppressedAlerts = ({ }; }); }; - -/** - * wraps suppressed alerts - * creates instanceId hash, which is used to search on time interval alerts - * populates alert's suppression fields - */ -export const wrapSuppressedSequenceAlerts = ({ - sequences, - spaceId, - completeRule, - mergeStrategy, - indicesToQuery, - buildReasonMessage, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - primaryTimestamp, - secondaryTimestamp, -}: { - sequences: Array>; - spaceId: string; - completeRule: CompleteRule; - mergeStrategy: ConfigType['alertMergeStrategy']; - indicesToQuery: string[]; - buildReasonMessage: BuildReasonMessage; - alertTimestampOverride: Date | undefined; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - publicBaseUrl: string | undefined; - primaryTimestamp: string; - secondaryTimestamp?: string; -}): Array< - WrappedFieldsLatest & { - subAlerts: Array>; - } -> => { - return sequences.reduce( - ( - acc: Array< - WrappedFieldsLatest & { - subAlerts: Array>; - } - >, - sequence - ) => { - // const alertGroupFromSequence = buildAlertGroupFromSequence({ - // ruleExecutionLogger, - // sequence, - // completeRule, - // mergeStrategy, - // spaceId, - // buildReasonMessage, - // indicesToQuery, - // alertTimestampOverride, - // applyOverrides: true, - // publicBaseUrl, - // }); - // const shellAlert = alertGroupFromSequence[0]; - // // have to check for null to satisfy type assertions below - // // otherwise i would still have to write shellAlert?.propert - // if (shellAlert == null || !isEqlShellAlert(shellAlert?._source)) { - // return [...acc]; - // } - // const buildingBlocks = alertGroupFromSequence.slice(1) as Array< - // WrappedFieldsLatest - // >; - // const suppressionTerms = getSuppressionTerms({ - // alertSuppression: completeRule?.ruleParams?.alertSuppression, - // fields: shellAlert?._source, - // }); - // const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - // const suppressionFields = getSuppressionAlertFields({ - // primaryTimestamp, - // secondaryTimestamp, - // // as casting should work because the alert fields are flattened (hopefully?) - // fields: shellAlert?._source as Record | undefined, - // suppressionTerms, - // fallbackTimestamp: alertTimestampOverride?.toISOString() ?? new Date().toISOString(), - // instanceId, - // }); - // const theFields = Object.keys(suppressionFields) as Array; - // // mutates shell alert to contain values from suppression fields - // theFields.forEach((field) => (shellAlert._source[field] = suppressionFields[field])); - // shellAlert.subAlerts = buildingBlocks; - // return [...acc, shellAlert] as Array< - // WrappedFieldsLatest & { - // subAlerts: Array>; - // } - // >; - return acc; - }, - [] as Array< - WrappedFieldsLatest & { - subAlerts: Array>; - } - > - ); -}; From cc7687c9558f32d9322491929565cd6a55351d53 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 7 Nov 2024 16:17:47 -0500 Subject: [PATCH 45/70] use latest fields for type guards --- .../rule_types/utils/utils.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index 3a76559b75116..ceb0f19a3cc80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -85,10 +85,6 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants'; import type { GenericBulkCreateResponse } from '../factories'; -import type { - EqlBuildingBlockAlert800, - EqlShellAlert800, -} from '../../../../../common/api/detection_engine/model/alerts/8.0.0'; import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; import type { ExtraFieldsForShellAlert, @@ -1058,13 +1054,15 @@ export const getDisabledActionsWarningText = ({ }; export const isEqlBuildingBlockAlert = ( - alertObject: unknown -): alertObject is EqlBuildingBlockAlert800 => - (alertObject as EqlBuildingBlockAlert800)?.[ALERT_BUILDING_BLOCK_TYPE] != null; - -export const isEqlShellAlert = (alertObject: unknown): alertObject is EqlShellAlert800 => - (alertObject as EqlShellAlert800)?.[ALERT_UUID] != null && - (alertObject as EqlShellAlert800)?.[ALERT_GROUP_ID] != null; + alertObject: BaseFieldsLatest +): alertObject is EqlBuildingBlockFieldsLatest => + (alertObject as EqlBuildingBlockFieldsLatest)?.[ALERT_BUILDING_BLOCK_TYPE] != null; + +export const isEqlShellAlert = ( + alertObject: BaseFieldsLatest +): alertObject is EqlShellFieldsLatest => + (alertObject as EqlShellFieldsLatest)?.[ALERT_UUID] != null && + (alertObject as EqlShellFieldsLatest)?.[ALERT_GROUP_ID] != null; export type RuleWithInMemorySuppression = | ThreatRuleParams From 74b01c9bbe56b03d82a8b3c084893132506e29ed Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 8 Nov 2024 11:03:03 -0500 Subject: [PATCH 46/70] Revert "prevent eql sequence suppression for building block rule types" This reverts commit 36f9b975813a279d1454214f09566cd9309e64b4. --- .../create_rule/request_schema_validation.ts | 13 ------------ .../patch_rule/request_schema_validation.ts | 15 -------------- .../update_rule/request_schema_validation.ts | 13 ------------ .../import_rules/rule_to_import_validation.ts | 19 +----------------- .../components/step_about_rule/index.tsx | 20 +------------------ .../pages/rule_creation/index.tsx | 11 ---------- .../pages/rule_editing/index.tsx | 12 +---------- .../detection_engine/rule_types/eql/eql.ts | 2 -- 8 files changed, 3 insertions(+), 102 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts index db8ccf3eddbe1..519ef874c422e 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/create_rule/request_schema_validation.ts @@ -5,7 +5,6 @@ * 2.0. */ -import isEmpty from 'lodash/isEmpty'; import type { RuleCreateProps } from '../../../model'; /** @@ -17,21 +16,9 @@ export const validateCreateRuleProps = (props: RuleCreateProps): string[] => { ...validateTimelineTitle(props), ...validateThreatMapping(props), ...validateThreshold(props), - ...validateSequenceSuppressionXorBuildingBlockRuleType(props), ]; }; -const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleCreateProps): string[] => { - if ( - props.type === 'eql' && - !isEmpty(props.building_block_type) && - !isEmpty(props.alert_suppression) - ) { - return ['rule cannot be an eql rule with building block type and have suppression enabled']; - } - return []; -}; - const validateTimelineId = (props: RuleCreateProps): string[] => { if (props.timeline_id != null) { if (props.timeline_title == null) { diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts index e3686e1d19d06..cb34c7fa8ecb5 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/patch_rule/request_schema_validation.ts @@ -5,7 +5,6 @@ * 2.0. */ -import isEmpty from 'lodash/isEmpty'; import type { PatchRuleRequestBody } from './patch_rule_route.gen'; /** @@ -17,23 +16,9 @@ export const validatePatchRuleRequestBody = (rule: PatchRuleRequestBody): string ...validateTimelineId(rule), ...validateTimelineTitle(rule), ...validateThreshold(rule), - ...validateSequenceSuppressionXorBuildingBlockRuleType(rule), ]; }; -const validateSequenceSuppressionXorBuildingBlockRuleType = ( - props: PatchRuleRequestBody -): string[] => { - if ( - props.type === 'eql' && - !isEmpty(props.building_block_type) && - !isEmpty(props.alert_suppression) - ) { - return ['rule cannot be an eql rule with building block type and have suppression enabled']; - } - return []; -}; - const validateId = (rule: PatchRuleRequestBody): string[] => { if (rule.id != null && rule.rule_id != null) { return ['both "id" and "rule_id" cannot exist, choose one or the other']; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts index a5c85d30a6c8b..d58bbdc4bee05 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/crud/update_rule/request_schema_validation.ts @@ -5,7 +5,6 @@ * 2.0. */ -import isEmpty from 'lodash/isEmpty'; import type { RuleUpdateProps } from '../../../model'; /** @@ -17,21 +16,9 @@ export const validateUpdateRuleProps = (props: RuleUpdateProps): string[] => { ...validateTimelineId(props), ...validateTimelineTitle(props), ...validateThreshold(props), - ...validateSequenceSuppressionXorBuildingBlockRuleType(props), ]; }; -const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleUpdateProps): string[] => { - if ( - props.type === 'eql' && - !isEmpty(props.building_block_type) && - !isEmpty(props.alert_suppression) - ) { - return ['rule cannot be an eql rule with building block type and have suppression enabled']; - } - return []; -}; - const validateId = (props: RuleUpdateProps): string[] => { if (props.id != null && props.rule_id != null) { return ['both "id" and "rule_id" cannot exist, choose one or the other']; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts index ac62f717e77ed..cef6d0fa03685 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts @@ -5,30 +5,13 @@ * 2.0. */ -import isEmpty from 'lodash/isEmpty'; import type { RuleToImport, ValidatedRuleToImport } from './rule_to_import'; /** * Additional validation that is implemented outside of the schema itself. */ export const validateRuleToImport = (rule: RuleToImport): string[] => { - return [ - ...validateTimelineId(rule), - ...validateTimelineTitle(rule), - ...validateThreshold(rule), - ...validateSequenceSuppressionXorBuildingBlockRuleType(rule), - ]; -}; - -const validateSequenceSuppressionXorBuildingBlockRuleType = (props: RuleToImport): string[] => { - if ( - props.type === 'eql' && - !isEmpty(props.building_block_type) && - !isEmpty(props.alert_suppression) - ) { - return ['rule cannot be an eql rule with building block type and have suppression enabled']; - } - return []; + return [...validateTimelineId(rule), ...validateTimelineTitle(rule), ...validateThreshold(rule)]; }; const validateTimelineId = (rule: RuleToImport): string[] => { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx index 972ceb93c4b34..7666a9ba8aee3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx @@ -55,7 +55,6 @@ interface StepAboutRuleProps extends RuleStepProps { timestampOverride: string; form: FormHook; esqlQuery?: string | undefined; - eqlSequenceSuppressionEnabled: boolean; } interface StepAboutRuleReadOnlyProps { @@ -86,7 +85,6 @@ const StepAboutRuleComponent: FC = ({ isLoading, form, esqlQuery, - eqlSequenceSuppressionEnabled, }) => { const { data } = useKibana().services; @@ -95,16 +93,6 @@ const StepAboutRuleComponent: FC = ({ const { ruleIndices } = useRuleIndices(machineLearningJobId, index); - const [buildingBlockRuleTypeChecked, setBuildingBlockRuleTypeChecked] = useState(false); - - const buildingBlockRuleTypeCheckedOnChange = (e: React.ChangeEvent) => { - if (eqlSequenceSuppressionEnabled) { - setBuildingBlockRuleTypeChecked(false); - } else { - setBuildingBlockRuleTypeChecked(e.target.checked); - } - }; - /** * 1. if not null, fetch data view from id saved on rule form * 2. Create a state to set the indexPattern to be used @@ -332,19 +320,13 @@ const StepAboutRuleComponent: FC = ({ /> - {/* do not display building block rule type option if eql sequence suppression is in use*/} ) => - buildingBlockRuleTypeCheckedOnChange(e), + disabled: isLoading, }, }} /> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx index 0d4dc6aebb3bd..0c6a6fb07ce5c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/index.tsx @@ -24,7 +24,6 @@ import { isMlRule, isThreatMatchRule, isEsqlRule, - isEqlSequenceQuery, } from '../../../../../common/detection_engine/utils'; import { useCreateRule } from '../../../rule_management/logic'; import type { RuleCreateProps } from '../../../../../common/api/detection_engine/model/rule_schema'; @@ -82,7 +81,6 @@ import { NextStep } from '../../components/next_step'; import { useRuleForms, useRuleFormsErrors, useRuleIndexPattern } from '../form'; import { CustomHeaderPageMemo } from '..'; import { SaveWithErrorsModal } from '../../components/save_with_errors_confirmation'; -import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; const MyEuiPanel = styled(EuiPanel)<{ zindex?: number; @@ -214,12 +212,6 @@ const CreateRulePageComponent: React.FC = () => { const esqlQueryForAboutStep = useEsqlQueryForAboutStep({ defineStepData, activeStep }); - const isEqlSeqQuery = isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string); - const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( - ruleType, - isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string) - ); - const esqlIndex = useEsqlIndex(defineStepData.queryBar.query.query, ruleType); const memoizedIndex = useMemo( @@ -664,7 +656,6 @@ const CreateRulePageComponent: React.FC = () => { isLoading={isCreateRuleLoading || loading} form={aboutStepForm} esqlQuery={esqlQueryForAboutStep} - eqlSequenceSuppressionEnabled={isAlertSuppressionEnabled && isEqlSeqQuery} /> { loading, memoAboutStepReadOnly, esqlQueryForAboutStep, - isAlertSuppressionEnabled, - isEqlSeqQuery, ] ); const memoAboutStepExtraAction = useMemo( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index 1a34feaa3132d..002e12c319a1e 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -22,7 +22,7 @@ import React, { memo, useCallback, useMemo, useRef, useState } from 'react'; import { useParams } from 'react-router-dom'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; -import { isEqlSequenceQuery, isEsqlRule } from '../../../../../common/detection_engine/utils'; +import { isEsqlRule } from '../../../../../common/detection_engine/utils'; import { RulePreview } from '../../components/rule_preview'; import { getIsRulePreviewDisabled } from '../../components/rule_preview/helpers'; import type { @@ -71,7 +71,6 @@ import { useRuleForms, useRuleFormsErrors, useRuleIndexPattern } from '../form'; import { useEsqlIndex, useEsqlQueryForAboutStep } from '../../hooks'; import { CustomHeaderPageMemo } from '..'; import { SaveWithErrorsModal } from '../../components/save_with_errors_confirmation'; -import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const { addSuccess } = useAppToasts(); @@ -144,12 +143,6 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const esqlIndex = useEsqlIndex(defineStepData.queryBar.query.query, defineStepData.ruleType); - const isEqlSeqQuery = isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string); - const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( - defineStepData.ruleType, - isEqlSequenceQuery(defineStepData.queryBar?.query?.query as string) - ); - const memoizedIndex = useMemo( () => (isEsqlRule(defineStepData.ruleType) ? esqlIndex : defineStepData.index), [defineStepData.index, esqlIndex, defineStepData.ruleType] @@ -286,7 +279,6 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { form={aboutStepForm} esqlQuery={esqlQueryForAboutStep} key="aboutStep" - eqlSequenceSuppressionEnabled={isAlertSuppressionEnabled && isEqlSeqQuery} /> )} @@ -375,8 +367,6 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { actionsStepForm, memoizedIndex, esqlQueryForAboutStep, - isAlertSuppressionEnabled, - isEqlSeqQuery, ] ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index 1e22d74e3519d..a74f2d17fac5e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -108,8 +108,6 @@ export const eqlExecutor = async ({ const isLoggedRequestsEnabled = state?.isLoggedRequestsEnabled ?? false; const loggedRequests: RulePreviewLoggedRequest[] = []; - // TODO: fix complexity warning - // eslint-disable-next-line complexity return withSecuritySpan('eqlExecutor', async () => { const result = createSearchAfterReturnType(); From 6ed81210ee503b640aeeab4d018023f27adeb5aa Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 8 Nov 2024 12:50:35 -0500 Subject: [PATCH 47/70] adds e2e test for suppressing alerts from building block rule type --- .../eql_alert_suppression.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 6c1af2da52af2..e8a0a2ce3cb79 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -1859,6 +1859,82 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('suppresses alerts when the rule is a building block type rule', async () => { + // hello world + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp2, + }; + + // sequence alert 1 is made up of doc1 and doc1WithLaterTimestamp, + // sequence alert 2 is made up of doc1WithLaterTimestamp and doc2WithLaterTimestamp + // sequence alert 2 is suppressed because it shares the same + // host.name value as sequence alert 1 + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithLaterTimestamp]); + + const buildingBlockRuleType: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + building_block_type: 'default', + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule: buildingBlockRuleType, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one created alert and one suppressed alert + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + // console.error(JSON.stringify(previewAlerts, null, 2)); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source[ALERT_SUPPRESSION_DOCS_COUNT] != null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + it('suppresses alerts in a given rule execution when a subsequent event for an sequence has the suppression field undefined', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; From b4aac4a632175bf860a8aed3711cdc551fdc03b1 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 8 Nov 2024 16:19:08 -0500 Subject: [PATCH 48/70] remove console log --- .../rule_types/utils/bulk_create_with_suppression.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index e9be216a2bf03..a1b0c5e1923c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -109,8 +109,6 @@ export const bulkCreateWithSuppression = async < : undefined, })); - console.error('DO WE HAVE ALERTS TO SUPPRESS?', alerts.length); - const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( alerts, From d6ca5b579603ddaa24bb83df0bd6ee8373e37518 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 12 Nov 2024 12:40:17 -0500 Subject: [PATCH 49/70] cleans up comments in test file, remove unused property leftover from revert of validation logic. --- .../components/step_about_rule/index.test.tsx | 1 - .../eql_alert_suppression.ts | 60 ++++++++----------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx index 9364764c7fd55..bdbc01ada58ff 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx @@ -110,7 +110,6 @@ describe('StepAboutRuleComponent', () => { timestampOverride={stepAboutDefaultValue.timestampOverride} isLoading={false} form={aboutStepForm} - eqlSequenceSuppressionEnabled={false} /> ); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index e8a0a2ce3cb79..fab56c4ae6d6c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -1860,7 +1860,6 @@ export default ({ getService }: FtrProviderContext) => { }); it('suppresses alerts when the rule is a building block type rule', async () => { - // hello world const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; const laterTimestamp = '2020-10-28T06:51:00.000Z'; @@ -1913,10 +1912,9 @@ export default ({ getService }: FtrProviderContext) => { // we expect one created alert and one suppressed alert // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); - // console.error(JSON.stringify(previewAlerts, null, 2)); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, - (alert) => alert?._source[ALERT_SUPPRESSION_DOCS_COUNT] != null + (alert) => alert?._source?.[ALERT_SUPPRESSION_DOCS_COUNT] != null ); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -1935,7 +1933,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('suppresses alerts in a given rule execution when a subsequent event for an sequence has the suppression field undefined', async () => { + it('suppresses alerts in a given rule execution when a subsequent event for a sequence has the suppression field undefined', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; const laterTimestamp = '2020-10-28T06:51:00.000Z'; @@ -1958,9 +1956,10 @@ export default ({ getService }: FtrProviderContext) => { // sequence alert 1 will be doc1 and doc1WithLaterTimestamp // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost - // the reason for the second alert is because despite the value being null - // in one of the two events in the sequence, the sequence alert will - // adopt the value for host.name and be suppressible. + // the second sequence alert will have null as the value for + // suppression field host.name because of logic defined in the + // objectPairIntersection function defined in + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts await indexListOfSourceDocuments([ doc1, @@ -1991,8 +1990,8 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - // we expect one alert and one suppressed alerts - // and two building block alerts, let's confirm that + // we expect two sequence alerts + // each sequence alert having two building block alerts expect(previewAlerts.length).toEqual(6); const [sequenceAlerts, buildingBlockAlerts] = partition( previewAlerts, @@ -2027,11 +2026,13 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('suppresses alerts in a given rule execution when doNotSuppress is set and one event in the sequence has the suppression field undefined', async () => { + it('does not suppress alerts in a given rule execution when doNotSuppress is set and more than one sequence has the suppression field undefined', async () => { const id = uuidv4(); const timestamp = '2020-10-28T06:50:00.000Z'; const laterTimestamp = '2020-10-28T06:51:00.000Z'; const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + const doc1 = { id, '@timestamp': timestamp, @@ -2048,12 +2049,23 @@ export default ({ getService }: FtrProviderContext) => { host: undefined, }; + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: undefined, + }; + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost // the reason for the second alert is because despite the value being null // in one of the two events in the sequence, the sequence alert will // adopt the value for host.name and be suppressible. - await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithNoHost]); + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithNoHost, + doc3WithNoHost, + ]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), @@ -2079,13 +2091,12 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and one suppressed alert // and two building block alerts per shell alert, let's confirm that - // expect(previewAlerts.length).toEqual(6); const [sequenceAlerts, buildingBlockAlerts] = partition( previewAlerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null ); - expect(buildingBlockAlerts.length).toEqual(4); - expect(sequenceAlerts.length).toEqual(2); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlerts.length).toEqual(3); const [suppressedSequenceAlerts] = partition( sequenceAlerts, (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 @@ -2153,7 +2164,6 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that - // console.error(JSON.stringify(previewAlerts, null, 2)); expect(previewAlerts.length).toEqual(6); const [sequenceAlert, buildingBlockAlerts] = partition( previewAlerts, @@ -2790,7 +2800,6 @@ export default ({ getService }: FtrProviderContext) => { ); // for sequence alerts if neither of the fields are there, we cannot suppress - // expect(sequenceAlert.length).toEqual(10); const [suppressedSequenceAlerts] = partition( sequenceAlert, (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 @@ -2917,8 +2926,6 @@ export default ({ getService }: FtrProviderContext) => { const createdRule = await createRule(supertest, log, rule); const alerts = await getOpenAlerts(supertest, log, es, createdRule); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that expect(alerts.hits.hits.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( alerts.hits.hits, @@ -2985,8 +2992,6 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-a', }, ], - // [ALERT_SUPPRESSION_START]: firstTimestamp2, // suppression start is the same - // [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed }); // suppression end value should be greater than second document timestamp, but lesser than current time @@ -3032,8 +3037,6 @@ export default ({ getService }: FtrProviderContext) => { const createdRule = await createRule(supertest, log, rule); const alerts = await getOpenAlerts(supertest, log, es, createdRule); - // we expect one alert and two suppressed alerts - // and two building block alerts, let's confirm that expect(alerts.hits.hits.length).toEqual(3); const [sequenceAlert, buildingBlockAlerts] = partition( alerts.hits.hits, @@ -3042,9 +3045,6 @@ export default ({ getService }: FtrProviderContext) => { expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); - // suppression start equal to alert timestamp - const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; - expect(sequenceAlert[0]._source).toEqual({ ...sequenceAlert[0]._source, [ALERT_SUPPRESSION_TERMS]: [ @@ -3053,10 +3053,6 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-a', }, ], - // suppression boundaries equal to original event time, since no alert been suppressed - // [ALERT_SUPPRESSION_START]: firstTimestamp, - // [ALERT_SUPPRESSION_END]: firstTimestamp, - // [TIMESTAMP]: suppressionStart, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }); @@ -3104,8 +3100,6 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-a', }, ], - // [ALERT_SUPPRESSION_START]: firstTimestamp, - // [ALERT_SUPPRESSION_END]: firstTimestamp, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed }); expect(sequenceAlert2[1]._source).toEqual({ @@ -3116,8 +3110,6 @@ export default ({ getService }: FtrProviderContext) => { value: 'host-a', }, ], - // [ALERT_SUPPRESSION_START]: secondTimestamp2, - // [ALERT_SUPPRESSION_END]: secondTimestamp2, [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed }); }); @@ -3204,7 +3196,7 @@ export default ({ getService }: FtrProviderContext) => { { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, ]); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp2 = new Date(dateNow2 + 15000); + const afterTimestamp2 = new Date(dateNow2 + 25000); const secondAlerts = await getOpenAlerts( supertest, log, From 9d839586c08f671ce82265223db518c97f2b1d5d Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 10:07:37 -0500 Subject: [PATCH 50/70] flatten subAlerts when present --- .../utils/create_persistence_rule_type_wrapper.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index fcc38a5a6bc4a..ec736d4d63494 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -550,10 +550,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper // we can now augment and enrich // the sub alerts (if any) the same as we would // any other newAlert - let enrichedAlerts = newAlerts.reduce((acc, newAlert) => { - const { subAlerts, ...everything } = newAlert; - return [...acc, everything, ...(subAlerts ?? [])]; - }, [] as typeof newAlerts); + let enrichedAlerts = newAlerts.some((newAlert) => newAlert.subAlerts != null) + ? newAlerts.flatMap((newAlert) => { + const { subAlerts, ...everything } = newAlert; + return [everything, ...(subAlerts ?? [])]; + }) + : newAlerts; if (enrichAlerts) { try { From 002500dd09190b29b66c20f3ba5e0651a014e677 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 10:12:33 -0500 Subject: [PATCH 51/70] use data already present --- .../rule_creation_ui/pages/rule_editing/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index 7db00e622a1ac..e7615e52c09b7 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -377,7 +377,7 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const saveChanges = useCallback(async () => { startTransaction({ name: SINGLE_RULE_ACTIONS.SAVE }); const localDefineStepData: DefineStepRule = defineFieldsTransform({ - ...defineStepForm.getFormData(), + ...defineStepData, eqlOptions: eqlOptionsSelected, }); const updatedRule = await updateRule({ @@ -400,7 +400,7 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { }, [ aboutStepData, actionsStepData, - defineStepForm, + defineStepData, defineFieldsTransform, eqlOptionsSelected, addSuccess, From 5fad4543af8d12d1fbd3d3144d9ebd1e8bb32d9a Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 10:24:40 -0500 Subject: [PATCH 52/70] remove casting --- .../rule_types/eql/wrap_sequences_factory.ts | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index 2aa77bcc2224d..db34230b30674 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -39,31 +39,24 @@ export const wrapSequencesFactory = intendedTimestamp: Date | undefined; }): WrapSequences => (sequences, buildReasonMessage) => - sequences.reduce( - ( - acc: Array< - | WrappedFieldsLatest - | WrappedFieldsLatest - >, - sequence - ) => { - const alerts = buildAlertGroupFromSequence({ - ruleExecutionLogger, - sequence, - completeRule, - mergeStrategy, - spaceId, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - publicBaseUrl, - intendedTimestamp, - }); + sequences.reduce< + Array< + | WrappedFieldsLatest + | WrappedFieldsLatest + > + >((acc, sequence) => { + const alerts = buildAlertGroupFromSequence({ + ruleExecutionLogger, + sequence, + completeRule, + mergeStrategy, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl, + intendedTimestamp, + }); - return [...acc, ...alerts] as Array< - | WrappedFieldsLatest - | WrappedFieldsLatest - >; - }, - [] - ); + return [...acc, ...alerts]; + }, []); From 479b16a24e7e16bfdea473b6a80c32d44dd6ff7b Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 13:21:36 -0500 Subject: [PATCH 53/70] removes eql utils class, pass shared params to other functions, remove use of factories for sequence suppression, updates tests --- .../build_alert_group_from_sequence.test.ts | 24 +-- .../eql/build_alert_group_from_sequence.ts | 13 +- .../rule_types/eql/create_eql_alert_type.ts | 33 +++- .../rule_types/eql/eql.test.ts | 30 +++- .../detection_engine/rule_types/eql/eql.ts | 14 +- .../rule_types/eql/eql_utils.ts | 148 ------------------ .../rule_types/eql/wrap_sequences_factory.ts | 9 +- ...bulk_create_suppressed_alerts_in_memory.ts | 51 +++--- .../utils/bulk_create_with_suppression.ts | 2 - .../rule_types/utils/utils.ts | 33 ++-- .../utils/wrap_suppressed_alerts.ts | 24 +-- .../eql_alert_suppression.ts | 3 +- 12 files changed, 131 insertions(+), 253 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts index 70778385396c0..5f6d62ebd0d22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts @@ -43,7 +43,7 @@ describe('buildAlert', () => { sampleDocNoSortId('619389b2-b077-400e-b40b-abde20d675d3'), ], }; - const alertGroup = buildAlertGroupFromSequence({ + const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({ ruleExecutionLogger: ruleExecutionLoggerMock, sequence: eqlSequence, completeRule, @@ -54,8 +54,8 @@ describe('buildAlert', () => { alertTimestampOverride: undefined, publicBaseUrl: PUBLIC_BASE_URL, }); - expect(alertGroup.length).toEqual(3); - expect(alertGroup[0]).toEqual( + expect(buildingBlocks.length).toEqual(2); + expect(shellAlert).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -72,10 +72,10 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[0]?._source?.[ALERT_URL]).toContain( + expect(shellAlert?._source?.[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/f2db3574eaf8450e3f4d1cf4f416d70b110b035ae0a7a00026242df07f0a6c90?index=.alerts-security.alerts-space' ); - expect(alertGroup[1]).toEqual( + expect(buildingBlocks[0]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -92,10 +92,10 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[1]._source[ALERT_URL]).toContain( + expect(buildingBlocks[0]._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1dbc416333244efbda833832eb83f13ea5d980a33c2f981ca8d2b35d82a045da?index=.alerts-security.alerts-space' ); - expect(alertGroup[2]).toEqual( + expect(buildingBlocks[1]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: expect.arrayContaining([ @@ -113,14 +113,14 @@ describe('buildAlert', () => { }, { depth: 1, - id: alertGroup[0]?._id, + id: buildingBlocks[0]?._id, index: '', rule: sampleRuleGuid, type: 'signal', }, { depth: 1, - id: alertGroup[1]._id, + id: buildingBlocks[0]._id, index: '', rule: sampleRuleGuid, type: 'signal', @@ -132,12 +132,12 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[2]._source[ALERT_URL]).toContain( + expect(buildingBlocks[1]._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1b7d06954e74257140f3bf73f139078483f9658fe829fd806cc307fc0388fb23?index=.alerts-security.alerts-space' ); - const groupIds = alertGroup.map((alert) => alert?._source?.[ALERT_GROUP_ID]); + const groupIds = buildingBlocks.map((alert) => alert?._source?.[ALERT_GROUP_ID]); for (const groupId of groupIds) { - expect(groupId).toEqual(groupIds[0]); + expect(groupId).toEqual(shellAlert?._source?.[ALERT_GROUP_ID]); } }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 0541d4ba4c640..f7edfd0e9385d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -95,12 +95,13 @@ export const buildAlertGroupFromSequence = ({ alertTimestampOverride, publicBaseUrl, intendedTimestamp, -}: BuildAlertGroupFromSequence): Array< - WrappedFieldsLatest -> => { +}: BuildAlertGroupFromSequence): { + shellAlert: WrappedFieldsLatest | undefined; + buildingBlocks: Array>; +} => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { - return []; + return { shellAlert: undefined, buildingBlocks: [] }; } // The "building block" alerts start out as regular BaseFields. @@ -129,7 +130,7 @@ export const buildAlertGroupFromSequence = ({ ); } catch (error) { ruleExecutionLogger.error(error); - return []; + return { shellAlert: undefined, buildingBlocks: [] }; } // The ID of each building block alert depends on all of the other building blocks as well, @@ -190,7 +191,7 @@ export const buildAlertGroupFromSequence = ({ ); // sequence alert guaranteed to be first - return [sequenceAlert, ...wrappedBuildingBlocks]; + return { shellAlert: sequenceAlert, buildingBlocks: wrappedBuildingBlocks }; }; export interface BuildAlertRootParams { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index b8c672c88af44..79bc9c9849da1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -11,10 +11,12 @@ import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; import { SERVER_APP_ID } from '../../../../../common/constants'; import { EqlRuleParams } from '../../rule_schema'; import { eqlExecutor } from './eql'; -import type { CreateRuleOptions, SecurityAlertType } from '../types'; +import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types'; import { validateIndexPatterns } from '../utils'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; -import { EqlUtils } from './eql_utils'; +import type { SharedParams } from '../utils/utils'; +import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; +import type { BuildReasonMessage } from '../utils/reason_formatters'; export const createEqlAlertType = ( createOptions: CreateRuleOptions @@ -90,7 +92,7 @@ export const createEqlAlertType = ( licensing, }); - const eqlUtils = new EqlUtils({ + const sharedParams: SharedParams = { spaceId, completeRule, mergeStrategy, @@ -101,7 +103,27 @@ export const createEqlAlertType = ( primaryTimestamp, secondaryTimestamp, intendedTimestamp, - }); + }; + + const wrapSuppressedHits = ( + events: SignalSourceHit[], + buildReasonMessage: BuildReasonMessage + ) => + wrapSuppressedAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + buildReasonMessage, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + intendedTimestamp, + }); + const { result, loggedRequests } = await eqlExecutor({ completeRule, tuple, @@ -117,7 +139,8 @@ export const createEqlAlertType = ( secondaryTimestamp, exceptionFilter, unprocessedExceptions, - eqlUtils, + wrapSuppressedHits, + sharedParams, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index 3df3d9a291330..f1fabcfd0f96b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -18,7 +18,7 @@ import { getCompleteRuleMock, getEqlRuleParams } from '../../rule_schema/mocks'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; import { eqlExecutor } from './eql'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; -import type { IEqlUtils } from './eql_utils'; +import type { SharedParams } from '../utils/utils'; jest.mock('../../routes/index/get_index_version'); jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() })); @@ -39,7 +39,21 @@ describe('eql_executor', () => { }; const mockExperimentalFeatures = {} as ExperimentalFeatures; const mockScheduleNotificationResponseActionsService = jest.fn(); - const mockEqlUtils = {} as IEqlUtils; + const ruleExecutionLoggerMock = ruleExecutionLogMock.forExecutors.create(); + const SPACE_ID = 'space'; + const PUBLIC_BASE_URL = 'http://testkibanabaseurl.com'; + + const sharedParams: SharedParams = { + ruleExecutionLogger: ruleExecutionLoggerMock, + completeRule: eqlCompleteRule, + mergeStrategy: 'allFields', + spaceId: SPACE_ID, + indicesToQuery: eqlCompleteRule.ruleParams.index as string[], + alertTimestampOverride: undefined, + publicBaseUrl: PUBLIC_BASE_URL, + intendedTimestamp: undefined, + primaryTimestamp: new Date().toISOString(), + }; beforeEach(() => { jest.clearAllMocks(); @@ -70,8 +84,9 @@ describe('eql_executor', () => { primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [getExceptionListItemSchemaMock()], + wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, - eqlUtils: mockEqlUtils, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, experimentalFeatures: mockExperimentalFeatures, @@ -106,7 +121,8 @@ describe('eql_executor', () => { primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - eqlUtils: mockEqlUtils, + wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, @@ -131,7 +147,8 @@ describe('eql_executor', () => { primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - eqlUtils: mockEqlUtils, + wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, @@ -171,7 +188,8 @@ describe('eql_executor', () => { primaryTimestamp: '@timestamp', exceptionFilter: undefined, unprocessedExceptions: [], - eqlUtils: mockEqlUtils, + wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index a74f2d17fac5e..d1d241879c419 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -27,7 +27,9 @@ import type { SearchAfterAndBulkCreateReturnType, SignalSource, CreateRuleOptions, + WrapSuppressedHits, } from '../types'; +import type { SharedParams } from '../utils/utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -52,7 +54,6 @@ import { getDataTierFilter } from '../utils/get_data_tier_filter'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logEqlRequest } from '../utils/logged_requests'; import * as i18n from '../translations'; -import type { IEqlUtils } from './eql_utils'; interface EqlExecutorParams { inputIndex: string[]; @@ -64,12 +65,13 @@ interface EqlExecutorParams { version: string; bulkCreate: BulkCreate; wrapHits: WrapHits; - eqlUtils: IEqlUtils; + sharedParams: SharedParams; wrapSequences: WrapSequences; primaryTimestamp: string; secondaryTimestamp?: string; exceptionFilter: Filter | undefined; unprocessedExceptions: ExceptionListItemSchema[]; + wrapSuppressedHits: WrapSuppressedHits; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; isAlertSuppressionActive: boolean; @@ -93,7 +95,8 @@ export const eqlExecutor = async ({ secondaryTimestamp, exceptionFilter, unprocessedExceptions, - eqlUtils, + wrapSuppressedHits, + sharedParams, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, @@ -108,6 +111,7 @@ export const eqlExecutor = async ({ const isLoggedRequestsEnabled = state?.isLoggedRequestsEnabled ?? false; const loggedRequests: RulePreviewLoggedRequest[] = []; + // eslint-disable-next-line complexity return withSecuritySpan('eqlExecutor', async () => { const result = createSearchAfterReturnType(); @@ -174,7 +178,7 @@ export const eqlExecutor = async ({ ruleExecutionLogger, tuple, alertSuppression: completeRule.ruleParams.alertSuppression, - wrapSuppressedHits: eqlUtils.wrapSuppressedHits, + wrapSuppressedHits, alertTimestampOverride, alertWithSuppression, experimentalFeatures, @@ -196,7 +200,7 @@ export const eqlExecutor = async ({ ruleExecutionLogger, tuple, alertSuppression: completeRule.ruleParams.alertSuppression, - eqlUtils, + sharedParams, alertTimestampOverride, alertWithSuppression, experimentalFeatures, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts deleted file mode 100644 index b064bcf309505..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql_utils.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; -import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import type { - BaseFieldsLatest, - EqlBuildingBlockFieldsLatest, - EqlShellFieldsLatest, - WrappedFieldsLatest, -} from '../../../../../common/api/detection_engine/model/alerts'; -import type { BuildReasonMessage } from '../utils/reason_formatters'; -import type { RuleWithInMemorySuppression } from '../utils/utils'; -import { buildShellAlertSuppressionTermsAndFields } from '../utils/utils'; -import type { - BuildAlertGroupFromSequenceReturnType, - WrappedEqlShellOptionalSubAlertsType, -} from './build_alert_group_from_sequence'; -import { buildAlertGroupFromSequence } from './build_alert_group_from_sequence'; -import type { SignalSource, SignalSourceHit } from '../types'; -import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; -import type { CompleteRule } from '../../rule_schema'; -import type { ConfigType } from '../../../../config'; -import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; - -interface ConstructorParams { - spaceId: string; - completeRule: CompleteRule; - mergeStrategy: ConfigType['alertMergeStrategy']; - indicesToQuery: string[]; - alertTimestampOverride: Date | undefined; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - publicBaseUrl: string | undefined; - primaryTimestamp: string; - secondaryTimestamp?: string; - intendedTimestamp?: Date; -} - -/** - * Interface for EqlUtils - */ -export interface IEqlUtils { - wrapSuppressedHits( - events: SignalSourceHit[], - buildReasonMessage: BuildReasonMessage - ): Array>; - alertGroupFromSequenceBuilder( - sequence: EqlHitsSequence, - buildReasonMessage: BuildReasonMessage - ): BuildAlertGroupFromSequenceReturnType; - getShellAlertWithSuppressionTermsAndFields( - shellAlert: WrappedEqlShellOptionalSubAlertsType, - buildingBlockAlerts: Array> - ): WrappedFieldsLatest & { - subAlerts: Array>; - }; -} - -export class EqlUtils implements IEqlUtils { - #spaceId: string; - #completeRule: CompleteRule; - #mergeStrategy: ConfigType['alertMergeStrategy']; - #indicesToQuery: string[]; - #alertTimestampOverride: Date | undefined; - #ruleExecutionLogger: IRuleExecutionLogForExecutors; - #publicBaseUrl: string | undefined; - #primaryTimestamp: string; - #secondaryTimestamp: string | undefined; - #intendedTimestamp: Date | undefined; - constructor({ - spaceId, - completeRule, - mergeStrategy, - indicesToQuery, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - primaryTimestamp, - secondaryTimestamp, - intendedTimestamp, - }: ConstructorParams) { - this.#spaceId = spaceId; - this.#completeRule = completeRule; - this.#mergeStrategy = mergeStrategy; - this.#indicesToQuery = indicesToQuery; - this.#alertTimestampOverride = alertTimestampOverride; - this.#ruleExecutionLogger = ruleExecutionLogger; - this.#publicBaseUrl = publicBaseUrl; - this.#primaryTimestamp = primaryTimestamp; - this.#secondaryTimestamp = secondaryTimestamp; - this.#intendedTimestamp = intendedTimestamp; - } - - public wrapSuppressedHits(events: SignalSourceHit[], buildReasonMessage: BuildReasonMessage) { - return wrapSuppressedAlerts({ - events, - buildReasonMessage, - spaceId: this.#spaceId, - completeRule: this.#completeRule, - mergeStrategy: this.#mergeStrategy, - indicesToQuery: this.#indicesToQuery, - alertTimestampOverride: this.#alertTimestampOverride, - ruleExecutionLogger: this.#ruleExecutionLogger, - publicBaseUrl: this.#publicBaseUrl, - primaryTimestamp: this.#primaryTimestamp, - secondaryTimestamp: this.#secondaryTimestamp, - intendedTimestamp: this.#intendedTimestamp, - }); - } - - public alertGroupFromSequenceBuilder( - sequence: EqlHitsSequence, - buildReasonMessage: BuildReasonMessage - ) { - return buildAlertGroupFromSequence({ - sequence, - buildReasonMessage, - applyOverrides: true, - spaceId: this.#spaceId, - completeRule: this.#completeRule, - mergeStrategy: this.#mergeStrategy, - indicesToQuery: this.#indicesToQuery, - alertTimestampOverride: this.#alertTimestampOverride, - ruleExecutionLogger: this.#ruleExecutionLogger, - publicBaseUrl: this.#publicBaseUrl, - }); - } - - public getShellAlertWithSuppressionTermsAndFields( - shellAlert: WrappedEqlShellOptionalSubAlertsType, - buildingBlockAlerts: Array> - ) { - return buildShellAlertSuppressionTermsAndFields({ - shellAlert, - buildingBlockAlerts, - spaceId: this.#spaceId, - completeRule: this.#completeRule, - indicesToQuery: this.#indicesToQuery, - alertTimestampOverride: this.#alertTimestampOverride, - primaryTimestamp: this.#primaryTimestamp, - secondaryTimestamp: this.#secondaryTimestamp, - }); - } -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index db34230b30674..d7ce37dd1feb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -45,7 +45,7 @@ export const wrapSequencesFactory = | WrappedFieldsLatest > >((acc, sequence) => { - const alerts = buildAlertGroupFromSequence({ + const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({ ruleExecutionLogger, sequence, completeRule, @@ -57,6 +57,9 @@ export const wrapSequencesFactory = publicBaseUrl, intendedTimestamp, }); - - return [...acc, ...alerts]; + if (shellAlert) { + acc.push(shellAlert, ...buildingBlocks); + return acc; + } + return acc; }, []); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 4071da60279ff..eb2aae653e3cd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -18,7 +18,8 @@ import type { WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; -import { addToSearchAfterReturn } from './utils'; +import type { SharedParams } from './utils'; +import { addToSearchAfterReturn, buildShellAlertSuppressionTermsAndFields } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; @@ -34,7 +35,7 @@ import type { WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import { robustGet } from './source_fields_merging/utils/robust_field_access'; -import type { IEqlUtils } from '../eql/eql_utils'; +import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -82,7 +83,7 @@ export interface BulkCreateSuppressedSequencesParams toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; maxNumberOfAlertsMultiplier?: number; - eqlUtils: IEqlUtils; + sharedParams: SharedParams; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -91,7 +92,6 @@ export interface BulkCreateSuppressedSequencesParams */ export const bulkCreateSuppressedAlertsInMemory = async ({ enrichedEvents, - buildingBlockAlerts, toReturn, wrapHits, bulkCreate, @@ -131,7 +131,6 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ return executeBulkCreateAlerts({ suppressibleWrappedDocs, unsuppressibleWrappedDocs, - buildingBlockAlerts, toReturn, bulkCreate, services, @@ -155,11 +154,11 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ toReturn, bulkCreate, services, - buildReasonMessage, ruleExecutionLogger, tuple, alertSuppression, - eqlUtils, + buildReasonMessage, + sharedParams, alertWithSuppression, alertTimestampOverride, experimentalFeatures, @@ -177,18 +176,15 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ const unsuppressibleWrappedDocs: Array> = []; sequences.forEach((sequence) => { - const alertGroupFromSequence = eqlUtils.alertGroupFromSequenceBuilder( + const alertGroupFromSequence = buildAlertGroupFromSequence({ sequence, - buildReasonMessage - ); - const shellAlert = alertGroupFromSequence?.[0]; - const buildingBlocks = alertGroupFromSequence - .slice(1) - .filter( - (alertGroup): alertGroup is WrappedFieldsLatest => - alertGroup != null - ); - if (shellAlert != null) { + applyOverrides: true, + buildReasonMessage, + ...sharedParams, + }); + const shellAlert = alertGroupFromSequence.shellAlert; + const buildingBlocks = alertGroupFromSequence.buildingBlocks; + if (shellAlert) { if (!suppressOnMissingFields) { // does the shell alert have all the suppression fields? const hasEverySuppressionField = (alertSuppression?.groupBy || []).every( @@ -196,19 +192,21 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ robustGet({ key: suppressionPath, document: shellAlert._source }) != null ); if (!hasEverySuppressionField) { - unsuppressibleWrappedDocs.push(...[shellAlert, ...buildingBlocks]); + unsuppressibleWrappedDocs.push(shellAlert, ...buildingBlocks); } else { - const wrappedWithSuppressionTerms = eqlUtils.getShellAlertWithSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({ shellAlert, - buildingBlocks - ); + buildingBlockAlerts: buildingBlocks, + ...sharedParams, + }); suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } } else { - const wrappedWithSuppressionTerms = eqlUtils.getShellAlertWithSuppressionTermsAndFields( + const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({ shellAlert, - buildingBlocks - ); + buildingBlockAlerts: buildingBlocks, + ...sharedParams, + }); suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); } } @@ -243,7 +241,6 @@ export interface ExecuteBulkCreateAlertsParams { unsuppressibleWrappedDocs: Array>; suppressibleWrappedDocs: Array>; - buildingBlockAlerts?: Array>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; maxNumberOfAlertsMultiplier?: number; @@ -257,7 +254,6 @@ export const executeBulkCreateAlerts = async < >({ unsuppressibleWrappedDocs, suppressibleWrappedDocs, - buildingBlockAlerts, toReturn, bulkCreate, services, @@ -296,7 +292,6 @@ export const executeBulkCreateAlerts = async < alertWithSuppression, ruleExecutionLogger, wrappedDocs: suppressibleWrappedDocs, - buildingBlockAlerts, services, suppressionWindow, alertTimestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index a1b0c5e1923c7..6c7228d6b78af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -43,7 +43,6 @@ export const bulkCreateWithSuppression = async < alertWithSuppression, ruleExecutionLogger, wrappedDocs, - buildingBlockAlerts, services, suppressionWindow, alertTimestampOverride, @@ -55,7 +54,6 @@ export const bulkCreateWithSuppression = async < alertWithSuppression: SuppressedAlertService; ruleExecutionLogger: IRuleExecutionLogForExecutors; wrappedDocs: Array & { subAlerts?: Array> }>; - buildingBlockAlerts?: Array>; services: RuleServices; suppressionWindow: string; alertTimestampOverride: Date | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index ceb0f19a3cc80..aa53f4748a324 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -5,7 +5,7 @@ * 2.0. */ import { createHash } from 'crypto'; -import { chunk, get, invert, isEmpty, partition } from 'lodash'; +import { chunk, get, invert, isEmpty, merge, partition } from 'lodash'; import moment from 'moment'; import objectHash from 'object-hash'; @@ -17,7 +17,6 @@ import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS, - ALERT_BUILDING_BLOCK_TYPE, TIMESTAMP, ALERT_INSTANCE_ID, ALERT_SUPPRESSION_DOCS_COUNT, @@ -85,7 +84,7 @@ import type { } from '../../../../../common/api/detection_engine/model/alerts'; import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants'; import type { GenericBulkCreateResponse } from '../factories'; -import { ALERT_GROUP_ID } from '../../../../../common/field_maps/field_names'; +import type { ConfigType } from '../../../../config'; import type { ExtraFieldsForShellAlert, WrappedEqlShellOptionalSubAlertsType, @@ -1053,16 +1052,18 @@ export const getDisabledActionsWarningText = ({ } }; -export const isEqlBuildingBlockAlert = ( - alertObject: BaseFieldsLatest -): alertObject is EqlBuildingBlockFieldsLatest => - (alertObject as EqlBuildingBlockFieldsLatest)?.[ALERT_BUILDING_BLOCK_TYPE] != null; - -export const isEqlShellAlert = ( - alertObject: BaseFieldsLatest -): alertObject is EqlShellFieldsLatest => - (alertObject as EqlShellFieldsLatest)?.[ALERT_UUID] != null && - (alertObject as EqlShellFieldsLatest)?.[ALERT_GROUP_ID] != null; +export interface SharedParams { + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; + intendedTimestamp: Date | undefined; +} export type RuleWithInMemorySuppression = | ThreatRuleParams @@ -1103,7 +1104,7 @@ export const buildShellAlertSuppressionTermsAndFields = ({ } => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: shellAlert?._source, + fields: shellAlert._source, }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); @@ -1134,10 +1135,12 @@ export const buildShellAlertSuppressionTermsAndFields = ({ [ALERT_SUPPRESSION_DOCS_COUNT]: 0, }; + merge(shellAlert._source, suppressionFields); + return { _id: shellAlert._id, _index: shellAlert._index, - _source: { ...shellAlert._source, ...suppressionFields }, + _source: shellAlert._source as EqlShellFieldsLatest & SuppressionFieldsLatest, subAlerts: buildingBlockAlerts, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index f11496bc140dd..beb52593f5e6d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -15,21 +15,13 @@ import type { BaseFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; -import type { ConfigType } from '../../../../config'; -import type { - CompleteRule, - EqlRuleParams, - MachineLearningRuleParams, - ThreatRuleParams, -} from '../../rule_schema'; -import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; + import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; +import type { SharedParams } from './utils'; import { generateId } from './utils'; import type { BuildReasonMessage } from './reason_formatters'; -type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; - /** * wraps suppressed alerts * creates instanceId hash, which is used to search on time interval alerts @@ -50,18 +42,8 @@ export const wrapSuppressedAlerts = ({ intendedTimestamp, }: { events: SignalSourceHit[]; - spaceId: string; - completeRule: CompleteRule; - mergeStrategy: ConfigType['alertMergeStrategy']; - indicesToQuery: string[]; buildReasonMessage: BuildReasonMessage; - alertTimestampOverride: Date | undefined; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - publicBaseUrl: string | undefined; - primaryTimestamp: string; - secondaryTimestamp?: string; - intendedTimestamp: Date | undefined; -}): Array> => { +} & SharedParams): Array> => { return events.map((event) => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index fab56c4ae6d6c..0e3ee0a53d2c3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -3116,7 +3116,6 @@ export default ({ getService }: FtrProviderContext) => { it('does not suppress alerts outside of duration when query with 3 sequences', async () => { const id = uuidv4(); - // this timestamp is 1 minute in the past const dateNow = Date.now(); const timestampSequenceEvent1 = new Date(dateNow - 5000).toISOString(); const timestampSequenceEvent2 = new Date(dateNow - 5500).toISOString(); @@ -3196,7 +3195,7 @@ export default ({ getService }: FtrProviderContext) => { { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, ]); await patchRule(supertest, log, { id: createdRule.id, enabled: true }); - const afterTimestamp2 = new Date(dateNow2 + 25000); + const afterTimestamp2 = new Date(dateNow2 + 55000); const secondAlerts = await getOpenAlerts( supertest, log, From 88e05e14d184d9a24d5575d993733ccd72e8bdba Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 14:42:40 -0500 Subject: [PATCH 54/70] assert correct functionality for sequence rule with 3 sequences in the query --- .../eql_alert_suppression.ts | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 0e3ee0a53d2c3..2e052e1e5e03c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -3114,6 +3114,87 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('suppresses alerts when query has 3 sequences', async () => { + // the first sequence alert is comprised of events [timestamp1, ...2, ...3] + // the other suppressed alerts will be made up of the following sequences + // [timestamp2, timestamp3, timestamp4] + // [timestamp3, timestamp4, timestamp5] + // [timestamp4, timestamp5, timestamp6] + const id = uuidv4(); + const dateNow = Date.now(); + const timestamp1 = new Date(dateNow - 5000).toISOString(); + const timestamp2 = new Date(dateNow - 5500).toISOString(); + const timestamp3 = new Date(dateNow - 5800).toISOString(); + const timestamp4 = new Date(dateNow - 6000).toISOString(); + const timestamp5 = new Date(dateNow - 6100).toISOString(); + const timestamp6 = new Date(dateNow - 6200).toISOString(); + + const firstSequenceEvent = { + id, + '@timestamp': timestamp1, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstSequenceEvent, + { ...firstSequenceEvent, '@timestamp': timestamp2 }, + { ...firstSequenceEvent, '@timestamp': timestamp3 }, + ]); + + const secondSequenceEvent = { + id, + '@timestamp': timestamp4, + host: { + name: 'host-a', + }, + }; + + await indexListOfSourceDocuments([ + secondSequenceEvent, + { ...secondSequenceEvent, '@timestamp': timestamp5 }, + { ...secondSequenceEvent, '@timestamp': timestamp6 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: `sequence [any where id == "${id}"] [any where id == "${id}"] [any where id == "${id}"]`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 10, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-30s', + interval: '10s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one shell alert + // and three building block alerts + expect(alerts.hits.hits.length).toEqual(4); + const [sequenceAlert, buildingBlockAlerts] = partition( + alerts.hits.hits, + (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + ); + expect(buildingBlockAlerts.length).toEqual(3); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + }); + it('does not suppress alerts outside of duration when query with 3 sequences', async () => { const id = uuidv4(); const dateNow = Date.now(); From 22e1416f3ee25390aa7209863ef5baf8b2743570 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 15:04:19 -0500 Subject: [PATCH 55/70] fix comment in test --- .../eql_alert_suppression.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 2e052e1e5e03c..dda090aa044f2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -2055,11 +2055,20 @@ export default ({ getService }: FtrProviderContext) => { host: undefined, }; + // this should generate three sequence alerts // sequence alert 1 will be doc1 and doc1WithLaterTimestamp // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost - // the reason for the second alert is because despite the value being null - // in one of the two events in the sequence, the sequence alert will - // adopt the value for host.name and be suppressible. + // sequence alert 3 will be doc2WithNoHost and doc3WithNoHost + // This logic is defined in objectPairIntersection + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + // Any sequence comprised of events where one field contains a value, followed by any number of + // events in the sequence where the value is null or undefined will have that field + // stripped from the sequence alert. So given that, we expect three alerts here + // The sequence comprised of [doc1, doc1WithLaterTimestamp] will have + // host.name and 'host-a' as the suppression values + // the other two sequence alerts comprised of [doc1WithLaterTimestamp, doc2WithNoHost] + // and [doc2WithNoHost, doc3WithNoHost] will have no suppression values set because + // of the logic outlined above await indexListOfSourceDocuments([ doc1, doc1WithLaterTimestamp, From fcf28d3346508b188f8a4654195e129ba9909407 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 15:34:34 -0500 Subject: [PATCH 56/70] adds partition utility function within test suite --- .../eql_alert_suppression.ts | 108 ++++++------------ 1 file changed, 36 insertions(+), 72 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index dda090aa044f2..3749653d99c67 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -10,7 +10,10 @@ import { v4 as uuidv4 } from 'uuid'; import sortBy from 'lodash/sortBy'; import partition from 'lodash/partition'; -import { EqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { + DetectionAlert, + EqlRuleCreateProps, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; import { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, @@ -24,6 +27,8 @@ import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_U import { getSuppressionMaxSignalsWarning as getSuppressionMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { RiskEnrichmentFields } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/enrichments/types'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, @@ -64,6 +69,10 @@ export default ({ getService }: FtrProviderContext) => { log, }); + const partitionSequenceBuildingBlocks = ( + alerts: Array> + ) => partition(alerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null); + // NOTE: Add to second quality gate after feature is GA describe('@ess @serverless Alert Suppression for EQL rules', () => { before(async () => { @@ -1838,10 +1847,7 @@ export default ({ getService }: FtrProviderContext) => { // we expect one created alert and one suppressed alert // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -1993,10 +1999,8 @@ export default ({ getService }: FtrProviderContext) => { // we expect two sequence alerts // each sequence alert having two building block alerts expect(previewAlerts.length).toEqual(6); - const [sequenceAlerts, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlerts, buildingBlockAlerts] = + partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(4); expect(sequenceAlerts.length).toEqual(2); @@ -2100,10 +2104,8 @@ export default ({ getService }: FtrProviderContext) => { }); // we expect one alert and one suppressed alert // and two building block alerts per shell alert, let's confirm that - const [sequenceAlerts, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlerts, buildingBlockAlerts] = + partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(6); expect(sequenceAlerts.length).toEqual(3); const [suppressedSequenceAlerts] = partition( @@ -2174,10 +2176,7 @@ export default ({ getService }: FtrProviderContext) => { // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(6); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); const [suppressedSequenceAlerts] = partition( sequenceAlert, (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 @@ -2250,10 +2249,7 @@ export default ({ getService }: FtrProviderContext) => { // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); const [suppressedSequenceAlerts] = partition( sequenceAlert, (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 @@ -2339,10 +2335,7 @@ export default ({ getService }: FtrProviderContext) => { }); expect(previewAlerts.length).toEqual(9); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); const [suppressedSequenceAlerts] = partition( sequenceAlert, (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 @@ -2411,10 +2404,7 @@ export default ({ getService }: FtrProviderContext) => { // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -2498,10 +2488,7 @@ export default ({ getService }: FtrProviderContext) => { // we expect one alert and two suppressed alerts // and two building block alerts, let's confirm that expect(previewAlerts.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -2559,10 +2546,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: [ALERT_ORIGINAL_TIME], }); - const [sequenceAlert] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); expect(previewAlerts.length).toEqual(3); // one sequence, two building block expect(sequenceAlert[0]._source).toEqual({ ...sequenceAlert[0]._source, @@ -2657,10 +2641,7 @@ export default ({ getService }: FtrProviderContext) => { size: 100, sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding }); - const [sequenceAlert] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); // for sequence alerts if neither of the fields are there, we cannot suppress expect(sequenceAlert.length).toEqual(4); @@ -2803,10 +2784,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, size: 100, }); - const [sequenceAlert] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); // for sequence alerts if neither of the fields are there, we cannot suppress const [suppressedSequenceAlerts] = partition( @@ -2880,10 +2858,7 @@ export default ({ getService }: FtrProviderContext) => { previewId, sort: ['host.name', ALERT_ORIGINAL_TIME], }); - const [sequenceAlert, buildingBlockAlerts] = partition( - previewAlerts, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -2936,9 +2911,8 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getOpenAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - alerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits ); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -2987,10 +2961,7 @@ export default ({ getService }: FtrProviderContext) => { afterTimestamp ); - const [sequenceAlert2] = partition( - secondAlerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert2] = partitionSequenceBuildingBlocks(secondAlerts.hits.hits); expect(sequenceAlert2.length).toEqual(1); expect(sequenceAlert2[0]._source).toEqual({ @@ -3047,9 +3018,8 @@ export default ({ getService }: FtrProviderContext) => { const alerts = await getOpenAlerts(supertest, log, es, createdRule); expect(alerts.hits.hits.length).toEqual(3); - const [sequenceAlert, buildingBlockAlerts] = partition( - alerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits ); expect(buildingBlockAlerts.length).toEqual(2); expect(sequenceAlert.length).toEqual(1); @@ -3095,10 +3065,7 @@ export default ({ getService }: FtrProviderContext) => { afterTimestamp ); - const [sequenceAlert2] = partition( - secondAlerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null - ); + const [sequenceAlert2] = partitionSequenceBuildingBlocks(secondAlerts.hits.hits); expect(sequenceAlert2.length).toEqual(2); expect(sequenceAlert2[0]._source).toEqual({ @@ -3185,9 +3152,8 @@ export default ({ getService }: FtrProviderContext) => { // we expect one shell alert // and three building block alerts expect(alerts.hits.hits.length).toEqual(4); - const [sequenceAlert, buildingBlockAlerts] = partition( - alerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits ); expect(buildingBlockAlerts.length).toEqual(3); expect(sequenceAlert.length).toEqual(1); @@ -3244,9 +3210,8 @@ export default ({ getService }: FtrProviderContext) => { // we expect one shell alert // and three building block alerts expect(alerts.hits.hits.length).toEqual(4); - const [sequenceAlert, buildingBlockAlerts] = partition( - alerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits ); expect(buildingBlockAlerts.length).toEqual(3); expect(sequenceAlert.length).toEqual(1); @@ -3296,9 +3261,8 @@ export default ({ getService }: FtrProviderContext) => { afterTimestamp2 ); - const [sequenceAlert2, buildingBlockAlerts2] = partition( - secondAlerts.hits.hits, - (alert) => alert?._source?.['kibana.alert.building_block_type'] == null + const [sequenceAlert2, buildingBlockAlerts2] = partitionSequenceBuildingBlocks( + secondAlerts.hits.hits ); // two sequence alerts because the second one happened From 19c525752554820d18f5e87b65dd4003840037c6 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 15:41:52 -0500 Subject: [PATCH 57/70] remove unused types --- .../rule_types/eql/build_alert_group_from_sequence.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index f7edfd0e9385d..f830cde275d52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -67,16 +67,6 @@ export type WrappedEqlShellOptionalSubAlertsType = WrappedFieldsLatest>; }; -export type BuildAlertGroupFromSequenceReturnType = [ - WrappedEqlShellOptionalSubAlertsType?, - ...Array> -]; - -export type AlertGroupFromSequenceBuilder = ( - sequence: EqlSequence, - buildReasonMessage: BuildReasonMessage -) => BuildAlertGroupFromSequenceReturnType; - /** * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals From fe390bb73bdb9adc340420d1285332d0fa16e8c0 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2024 16:01:42 -0500 Subject: [PATCH 58/70] some more cleanup --- .../rule_types/eql/build_alert_group_from_sequence.ts | 1 - .../server/lib/detection_engine/rule_types/eql/eql.ts | 3 --- .../utils/bulk_create_suppressed_alerts_in_memory.ts | 1 - .../rule_types/utils/partition_missing_fields_events.ts | 1 + 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index f830cde275d52..2e0948b1af4d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -180,7 +180,6 @@ export const buildAlertGroupFromSequence = ({ } ); - // sequence alert guaranteed to be first return { shellAlert: sequenceAlert, buildingBlocks: wrappedBuildingBlocks }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index d1d241879c419..ab57171b64f73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -208,9 +208,6 @@ export const eqlExecutor = async ({ } else { newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } - // once partitioned, we pass in the sequence alerts to check for suppression - // and then filter out the suppressable sequence alerts and the building - // block alerts associated with the suppressable sequence alerts. } else { throw new Error( 'eql query response should have either `sequences` or `events` but had neither' diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index eb2aae653e3cd..4b7eaa065cf85 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -59,7 +59,6 @@ export interface BulkCreateSuppressedAlertsParams | 'alertTimestampOverride' > { enrichedEvents: SignalSourceHit[]; - buildingBlockAlerts?: Array>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; mergeSourceAndFields?: boolean; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index ef6b2c96cbb87..901768fe5c773 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -34,6 +34,7 @@ export const partitionMissingFieldsEvents = < (event as SignalSourceHit)?._source || (event as { event: SignalSourceHit })?.event?._source; const fields = mergeSourceAndFields ? { ...sourceFields, ...eventFields } : eventFields; + const hasMissingFields = Object.keys(pick(fields, suppressedBy)).length < suppressedBy.length; return !hasMissingFields; From 7c1a53d0730790343889c9f6ecb9a03b82b5d525 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 18 Nov 2024 11:54:58 -0500 Subject: [PATCH 59/70] remove unused type --- .../server/lib/detection_engine/rule_types/types.ts | 9 --------- .../utils/bulk_create_suppressed_alerts_in_memory.ts | 2 -- 2 files changed, 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 4d2d4914e6314..94aa55e6b0dd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -353,15 +353,6 @@ export type WrapSuppressedHits = ( buildReasonMessage: BuildReasonMessage ) => Array>; -export type WrapSuppressedSequences = ( - sequences: Array>, - buildReasonMessage: BuildReasonMessage -) => Array< - WrappedFieldsLatest & { - subAlerts: Array>; - } ->; - export type WrapSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 4b7eaa065cf85..6d0f9ae39518b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -15,7 +15,6 @@ import type { WrapSuppressedHits, SignalSourceHit, SignalSource, - WrapSuppressedSequences, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; import type { SharedParams } from './utils'; @@ -39,7 +38,6 @@ import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_seque interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; - wrapSuppressedSequences: WrapSuppressedSequences; alertTimestampOverride: Date | undefined; alertWithSuppression: SuppressedAlertService; alertSuppression?: AlertSuppressionCamel; From aa9d5d36041a5524a9489b083d110b7b822324e0 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 18 Nov 2024 15:06:08 -0500 Subject: [PATCH 60/70] narrow type for alertSuppression param --- .../server/lib/detection_engine/rule_types/eql/eql.ts | 4 +++- .../utils/bulk_create_suppressed_alerts_in_memory.ts | 5 +++-- .../rule_types/utils/get_is_alert_suppression_active.ts | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index ab57171b64f73..756f220f06d55 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -54,6 +54,7 @@ import { getDataTierFilter } from '../utils/get_data_tier_filter'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logEqlRequest } from '../utils/logged_requests'; import * as i18n from '../translations'; +import { alertSuppressionTypeGuard } from '../utils/get_is_alert_suppression_active'; interface EqlExecutorParams { inputIndex: string[]; @@ -189,7 +190,8 @@ export const eqlExecutor = async ({ } else if (sequences) { if ( isAlertSuppressionActive && - experimentalFeatures.alertSuppressionForSequenceEqlRuleEnabled + experimentalFeatures.alertSuppressionForSequenceEqlRuleEnabled && + alertSuppressionTypeGuard(completeRule.ruleParams.alertSuppression) ) { await bulkCreateSuppressedSequencesInMemory({ sequences, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 6d0f9ae39518b..291b5cfa0d2f6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -81,6 +81,7 @@ export interface BulkCreateSuppressedSequencesParams experimentalFeatures: ExperimentalFeatures; maxNumberOfAlertsMultiplier?: number; sharedParams: SharedParams; + alertSuppression: AlertSuppressionCamel; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -162,7 +163,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ maxNumberOfAlertsMultiplier, }: BulkCreateSuppressedSequencesParams) => { const suppressOnMissingFields = - (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === + (alertSuppression.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === AlertSuppressionMissingFieldsStrategyEnum.suppress; const suppressibleWrappedSequences: Array< @@ -184,7 +185,7 @@ export const bulkCreateSuppressedSequencesInMemory = async ({ if (shellAlert) { if (!suppressOnMissingFields) { // does the shell alert have all the suppression fields? - const hasEverySuppressionField = (alertSuppression?.groupBy || []).every( + const hasEverySuppressionField = alertSuppression.groupBy.every( (suppressionPath) => robustGet({ key: suppressionPath, document: shellAlert._source }) != null ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts index 5ab06db3043af..263bd88bc1b0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts @@ -17,6 +17,10 @@ interface GetIsAlertSuppressionActiveParams { licensing: LicensingPluginSetup; } +export const alertSuppressionTypeGuard = ( + alertSuppression: AlertSuppressionCamel | undefined +): alertSuppression is AlertSuppressionCamel => Boolean(alertSuppression?.groupBy?.length); + /** * checks if alert suppression is active: * - rule should have alert suppression config @@ -32,7 +36,7 @@ export const getIsAlertSuppressionActive = async ({ return false; } - const isAlertSuppressionConfigured = Boolean(alertSuppression?.groupBy?.length); + const isAlertSuppressionConfigured = alertSuppressionTypeGuard(alertSuppression); if (!isAlertSuppressionConfigured) { return false; From dce9b92f5af95cb80560446e3721fbee75c788ff Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 18 Nov 2024 16:23:20 -0500 Subject: [PATCH 61/70] updates return type and field name for getSuppressionTerms, also changes alert schema for suppression terms field to be SearchTypes --- .../common/schemas/8.17.0/index.ts | 32 +++++++++++++++++++ .../rule_registry/common/schemas/index.ts | 6 ++-- .../esql/wrap_suppressed_esql_alerts.ts | 2 +- .../wrap_suppressed_new_terms_alerts.ts | 2 +- .../utils/robust_field_access.test.ts | 3 ++ .../utils/suppression_utils.test.ts | 28 +++++++++++++--- .../rule_types/utils/suppression_utils.ts | 19 ++++++----- .../rule_types/utils/utils.ts | 2 +- .../utils/wrap_suppressed_alerts.ts | 2 +- 9 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts diff --git a/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts new file mode 100644 index 0000000000000..cc1d73de3c4ae --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_SUPPRESSION_TERMS } from '@kbn/rule-data-utils'; +import { SearchTypes } from '@kbn/data-plugin/common'; +import { AlertWithCommonFields880 } from '../8.8.0'; + +import { SuppressionFields8130 } from '../8.13.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.13.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.13.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields8170 + extends Omit { + [ALERT_SUPPRESSION_TERMS]: Array<{ + field: string; + value: SearchTypes | null; + }>; +} + +export type AlertWithSuppressionFields8170 = AlertWithCommonFields880 & SuppressionFields8170; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index 5c168a4b899cc..5a94d250392ef 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -13,11 +13,11 @@ import type { CommonAlertFields880, } from './8.8.0'; -import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; +import type { AlertWithSuppressionFields8170, SuppressionFields8170 } from './8.17.0'; export type { - AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest, - SuppressionFields8130 as SuppressionFieldsLatest, + AlertWithSuppressionFields8170 as AlertWithSuppressionFieldsLatest, + SuppressionFields8170 as SuppressionFieldsLatest, CommonAlertFieldName880 as CommonAlertFieldNameLatest, CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest, CommonAlertFields880 as CommonAlertFieldsLatest, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts index b2d01e4d5ee7a..fe72aa566583a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -61,7 +61,7 @@ export const wrapSuppressedEsqlAlerts = ({ const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: combinedFields, + input: combinedFields, }); const id = generateAlertId({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts index fa3781b2a2e63..dc9a322301126 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts @@ -58,7 +58,7 @@ export const wrapSuppressedNewTermsAlerts = ({ const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: event.fields, + input: event.fields, }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts index 646068b59eec7..1b4b9e7d49174 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts @@ -34,6 +34,9 @@ describe('robust field access', () => { it('returns undefined if the key does not exist', () => { expect(robustGet({ key: 'a.b.c', document: { a: { b: 'my-value' } } })).toEqual(undefined); }); + it('returns an array if the key exists', () => { + expect(robustGet({ key: 'a.b', document: { a: { b: ['my-value'] } } })).toEqual(['my-value']); + }); }); describe('set', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts index 745dd08977520..46aa25d08d44a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts @@ -85,17 +85,37 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.name': 'localhost-1' }, + input: { 'host.name': 'localhost-1' }, }) ).toEqual([{ field: 'host.name', value: 'localhost-1' }]); }); + it('should return suppression terms when using source', () => { + expect( + getSuppressionTerms({ + alertSuppression: { + groupBy: ['host.name'], + }, + input: { host: { name: 'localhost-1' } }, + }) + ).toEqual([{ field: 'host.name', value: 'localhost-1' }]); + }); + it('should return suppression terms when using source and mixed notation', () => { + expect( + getSuppressionTerms({ + alertSuppression: { + groupBy: ['host.something.name'], + }, + input: { 'host.something': { name: 'localhost-1' } }, + }) + ).toEqual([{ field: 'host.something.name', value: 'localhost-1' }]); + }); it('should return suppression terms array when fields do not have matches', () => { expect( getSuppressionTerms({ alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.ip': '127.0.0.1' }, + input: { 'host.ip': '127.0.0.1' }, }) ).toEqual([{ field: 'host.name', value: null }]); }); @@ -105,7 +125,7 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.name': ['localhost-2', 'localhost-1'] }, + input: { 'host.name': ['localhost-2', 'localhost-1'] }, }) ).toEqual([{ field: 'host.name', value: ['localhost-1', 'localhost-2'] }]); }); @@ -115,7 +135,7 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name', 'host.ip'], }, - fields: { 'host.name': ['localhost-1'], 'agent.name': 'test', 'host.ip': '127.0.0.1' }, + input: { 'host.name': ['localhost-1'], 'agent.name': 'test', 'host.ip': '127.0.0.1' }, }) ).toEqual([ { field: 'host.name', value: ['localhost-1'] }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index f63ca89384920..0e066069665dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -5,7 +5,6 @@ * 2.0. */ -import pick from 'lodash/pick'; import get from 'lodash/get'; import sortBy from 'lodash/sortBy'; @@ -18,10 +17,12 @@ import { } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; +import type { SearchTypes } from '../../../../../common/detection_engine/types'; export interface SuppressionTerm { field: string; - value: string[] | number[] | null; + value: SearchTypes | null; } /** @@ -60,23 +61,21 @@ export const getSuppressionAlertFields = ({ }; /** + * generates values from a source event for the fields provided in the alertSuppression object + * @param alertSuppression {@link AlertSuppressionCamel} options defining how to suppress alerts + * @param input source data from either the _source of "fields" property on the event * returns an array of {@link SuppressionTerm}s by retrieving the appropriate field values based on the provided alertSuppression configuration */ export const getSuppressionTerms = ({ alertSuppression, - fields, + input, }: { - fields: Record | undefined; alertSuppression: AlertSuppressionCamel | undefined; + input: Record | undefined; }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - const suppressedProps = pick(fields, suppressedBy) as Record< - string, - string[] | number[] | undefined - >; - const suppressionTerms = suppressedBy.map((field) => { - const value = get(suppressedProps, field) ?? null; + const value = input != null ? robustGet({ document: input, key: field }) ?? null : null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; return { field, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index aa53f4748a324..083b3f9841fa4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -1104,7 +1104,7 @@ export const buildShellAlertSuppressionTermsAndFields = ({ } => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: shellAlert._source, + input: shellAlert._source, }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index beb52593f5e6d..6cb176ba70f77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -47,7 +47,7 @@ export const wrapSuppressedAlerts = ({ return events.map((event) => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: event.fields, + input: event.fields, }); const id = generateId( From d1067ae3056f4059a1b7dbb1a6e1690e5b5922b5 Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Mon, 18 Nov 2024 16:26:18 -0500 Subject: [PATCH 62/70] partition on group.index field in e2e test Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> --- .../eql/trial_license_complete_tier/eql_alert_suppression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 3749653d99c67..935b3cbe3776c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -71,7 +71,7 @@ export default ({ getService }: FtrProviderContext) => { const partitionSequenceBuildingBlocks = ( alerts: Array> - ) => partition(alerts, (alert) => alert?._source?.['kibana.alert.building_block_type'] == null); + ) => partition(alerts, (alert) => alert?._source?.['kibana.alert.group.index'] == null); // NOTE: Add to second quality gate after feature is GA describe('@ess @serverless Alert Suppression for EQL rules', () => { From 73e5f9a72329d8861d8e72bfe0e10db6ea5a8f99 Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Mon, 18 Nov 2024 16:27:32 -0500 Subject: [PATCH 63/70] partition on group.index field in e2e test Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> From ba05c66857b3d7857403a214724175b9603e3410 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 18 Nov 2024 17:30:39 -0500 Subject: [PATCH 64/70] adds cypress test to create eql sequence rule with suppression and moves deletion of preview index in e2e test to after function --- .../eql_alert_suppression.ts | 2 +- .../eql_rule_suppression_sequence.cy.ts | 90 ++++++++++++++++++- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 935b3cbe3776c..d933b1b7274d5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -81,6 +81,7 @@ export default ({ getService }: FtrProviderContext) => { after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + await esDeleteAllIndices('.preview.alerts*'); }); afterEach(async () => { @@ -89,7 +90,6 @@ export default ({ getService }: FtrProviderContext) => { '.alerts-security.alerts-*', ]); await deleteAllRules(supertest, log); - await esDeleteAllIndices('.preview.alerts*'); }); describe('non-sequence queries', () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts index b945f76b38310..440a26bcc707f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts @@ -8,11 +8,31 @@ import { getEqlSequenceRule } from '../../../../objects/rule'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; +import { getDetails } from '../../../../tasks/rule_details'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { fillDefineEqlRule, selectEqlRuleType } from '../../../../tasks/create_new_rule'; +import { + fillAlertSuppressionFields, + fillAboutRuleMinimumAndContinue, + createRuleWithoutEnabling, + skipScheduleRuleAction, + continueFromDefineStep, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, + fillDefineEqlRule, + selectEqlRuleType, +} from '../../../../tasks/create_new_rule'; -import { ALERT_SUPPRESSION_FIELDS_INPUT } from '../../../../screens/create_new_rule'; +import { + DEFINITION_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, + DETAILS_TITLE, +} from '../../../../screens/rule_details'; + +const SUPPRESS_BY_FIELDS = ['agent.type']; describe( 'Detection Rule Creation - EQL Rules - With Alert Suppression', @@ -41,8 +61,70 @@ describe( fillDefineEqlRule(rule); }); - it('displays the suppression fields', () => { - cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); + it('creates a rule with a "per rule execution" suppression duration', () => { + // selecting only suppression fields, the rest options would be default + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + }); + }); + + it('creates a rule with a "per time interval" suppression duration', () => { + const expectedSuppressByFields = SUPPRESS_BY_FIELDS.slice(0, 1); + + // fill suppress by fields and select non-default suppression options + fillAlertSuppressionFields(expectedSuppressByFields); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(45, 'm'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); }); }); } From 65b5a9cfea17535b696c8120d876c47e39fce7f4 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 22 Nov 2024 10:50:59 -0500 Subject: [PATCH 65/70] fixes misaligned assertions between shell alert and building block alerts, skip cypress test in serverless because of feature flag --- .../eql/build_alert_group_from_sequence.test.ts | 14 +++++++------- .../rule_creation/eql_rule_suppression.cy.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts index 5f6d62ebd0d22..a2b0d4f21fc78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts @@ -55,7 +55,7 @@ describe('buildAlert', () => { publicBaseUrl: PUBLIC_BASE_URL, }); expect(buildingBlocks.length).toEqual(2); - expect(shellAlert).toEqual( + expect(buildingBlocks[0]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -72,10 +72,10 @@ describe('buildAlert', () => { }), }) ); - expect(shellAlert?._source?.[ALERT_URL]).toContain( + expect(buildingBlocks[0]?._source?.[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/f2db3574eaf8450e3f4d1cf4f416d70b110b035ae0a7a00026242df07f0a6c90?index=.alerts-security.alerts-space' ); - expect(buildingBlocks[0]).toEqual( + expect(buildingBlocks[1]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -92,10 +92,10 @@ describe('buildAlert', () => { }), }) ); - expect(buildingBlocks[0]._source[ALERT_URL]).toContain( + expect(buildingBlocks[1]._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1dbc416333244efbda833832eb83f13ea5d980a33c2f981ca8d2b35d82a045da?index=.alerts-security.alerts-space' ); - expect(buildingBlocks[1]).toEqual( + expect(shellAlert).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: expect.arrayContaining([ @@ -132,12 +132,12 @@ describe('buildAlert', () => { }), }) ); - expect(buildingBlocks[1]._source[ALERT_URL]).toContain( + expect(shellAlert?._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1b7d06954e74257140f3bf73f139078483f9658fe829fd806cc307fc0388fb23?index=.alerts-security.alerts-space' ); const groupIds = buildingBlocks.map((alert) => alert?._source?.[ALERT_GROUP_ID]); for (const groupId of groupIds) { - expect(groupId).toEqual(shellAlert?._source?.[ALERT_GROUP_ID]); + expect(groupId).toEqual(groupIds[0]); } }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts index 31855c3efff15..8f4b6a10faeb8 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts @@ -38,7 +38,7 @@ const SUPPRESS_BY_FIELDS = ['agent.type']; describe( 'Detection Rule Creation - EQL Rules - With Alert Suppression', { - tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + tags: ['@ess', '@skipInServerlessMKI'], }, () => { describe('with non-sequence queries', () => { From 09be989131d3d5469768e93dbc81a8a71b8c3568 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 26 Nov 2024 16:32:58 -0500 Subject: [PATCH 66/70] removes suppression fields when saving and suppression is disabled --- .../use_experimental_feature_fields_transform.ts | 14 ++++++++++---- .../rule_creation_ui/pages/rule_editing/index.tsx | 2 -- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index a95106d5da817..d896fa676d31d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -9,6 +9,12 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; import { isEqlRule, isEqlSequenceQuery } from '../../../../../common/detection_engine/utils'; +import { + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, +} from '../../../rule_creation/components/alert_suppression_edit'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -30,10 +36,10 @@ export const useExperimentalFeatureFieldsTransform = = ({ rule }) => { startTransaction({ name: SINGLE_RULE_ACTIONS.SAVE }); const localDefineStepData: DefineStepRule = defineFieldsTransform({ ...defineStepData, - eqlOptions: eqlOptionsSelected, }); const updatedRule = await updateRule({ ...formatRule( @@ -401,7 +400,6 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { actionsStepData, defineStepData, defineFieldsTransform, - eqlOptionsSelected, addSuccess, navigateToApp, rule?.exceptions_list, From cc27026e1cf6519eeacf7b950a41b3339e3d20a8 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Wed, 27 Nov 2024 07:21:14 -0500 Subject: [PATCH 67/70] skip test in serverless env because of feature flags --- .../rule_creation/eql_rule_suppression_sequence.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts index 440a26bcc707f..2cf8526a2f9c3 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts @@ -39,7 +39,7 @@ describe( { // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config - tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + tags: ['@ess', '@skipInServerlessMKI'], env: { kbnServerArgs: [ `--xpack.securitySolution.enableExperimental=${JSON.stringify([ From 5bcdf9f1c7170bd47a32989ac34a0243c1eee866 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 2 Dec 2024 12:12:24 -0500 Subject: [PATCH 68/70] eql sequence suppression feature flag set default to enabled --- .../plugins/security_solution/common/experimental_features.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 6721945cdce35..095324840fc5c 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -21,7 +21,7 @@ export const allowedExperimentalValues = Object.freeze({ * Turned: on (TBD) * Expires: on (TBD) */ - alertSuppressionForSequenceEqlRuleEnabled: false, + alertSuppressionForSequenceEqlRuleEnabled: true, // FIXME:PT delete? excludePoliciesInFilterEnabled: false, From a38142f4a9278e78d84f266200d25c847b001dee Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 3 Dec 2024 19:37:14 -0500 Subject: [PATCH 69/70] remove ruletype param from useAlertSuppression hook since all rule types support suppression, with eql sequence dependent on feature flag, so that is now the only parameter necessary --- .../components/step_define_rule/index.tsx | 1 - .../rule_details/rule_definition_section.tsx | 2 +- .../logic/use_alert_suppression.test.tsx | 34 +++++-------------- .../logic/use_alert_suppression.tsx | 17 +++------- 4 files changed, 13 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index a3c8be10df694..7b056b5969799 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -357,7 +357,6 @@ const StepDefineRuleComponent: FC = ({ const areSuppressionFieldsSelected = isThresholdRule || Boolean(alertSuppressionFields?.length); const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( - ruleType, isEqlSequenceQuery(queryBar?.query?.query as string) ); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index 70f267ac94ba4..3e08f4ce3acc8 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -843,7 +843,7 @@ export const RuleDefinitionSection = ({ ruleType: rule.type, }); - const { isSuppressionEnabled } = useAlertSuppression(rule.type); + const { isSuppressionEnabled } = useAlertSuppression(); const definitionSectionListItems = prepareDefinitionSectionListItems( rule, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index eba3850c35dfa..7877a86385cde 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -6,7 +6,6 @@ */ import { renderHook } from '@testing-library/react'; -import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import * as useIsExperimentalFeatureEnabledMock from '../../../common/hooks/use_experimental_features'; import { useAlertSuppression } from './use_alert_suppression'; @@ -14,33 +13,16 @@ describe('useAlertSuppression', () => { jest .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') .mockReturnValue(false); - ( - [ - 'new_terms', - 'threat_match', - 'saved_query', - 'query', - 'threshold', - 'eql', - 'esql', - 'machine_learning', - ] as Type[] - ).forEach((ruleType) => { - it(`should return the isSuppressionEnabled true for ${ruleType} rule type that exists in SUPPRESSIBLE_ALERT_RULES`, () => { - const { result } = renderHook(() => useAlertSuppression(ruleType)); - - expect(result.current.isSuppressionEnabled).toBe(true); - }); - }); - - it('should return false if rule type is undefined', () => { - const { result } = renderHook(() => useAlertSuppression(undefined)); - expect(result.current.isSuppressionEnabled).toBe(false); + it(`should return the isSuppressionEnabled true if query for all rule types is not an eql sequence query`, () => { + const { result } = renderHook(() => useAlertSuppression()); + expect(result.current.isSuppressionEnabled).toBe(true); }); - it('should return false if rule type is not a suppressible rule', () => { - const { result } = renderHook(() => useAlertSuppression('OTHER_RULE_TYPE' as Type)); - + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockReturnValue(false); + it('should return isSuppressionEnabled false for eql sequence query when feature flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression(true)); expect(result.current.isSuppressionEnabled).toBe(false); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index e6159494b75fe..dae422a8e81c5 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -5,33 +5,24 @@ * 2.0. */ import { useCallback } from 'react'; -import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { isEqlRule, isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } -export const useAlertSuppression = ( - ruleType: Type | undefined, - isEqlSequenceQuery = false -): UseAlertSuppressionReturn => { +export const useAlertSuppression = (isEqlSequenceQuery = false): UseAlertSuppressionReturn => { const isAlertSuppressionForSequenceEQLRuleEnabled = useIsExperimentalFeatureEnabled( 'alertSuppressionForSequenceEqlRuleEnabled' ); const isSuppressionEnabledForRuleType = useCallback(() => { - if (!ruleType) { - return false; - } - - if (isEqlRule(ruleType) && isEqlSequenceQuery) { + if (isEqlSequenceQuery) { return isAlertSuppressionForSequenceEQLRuleEnabled; } - return isSuppressibleAlertRule(ruleType); - }, [ruleType, isAlertSuppressionForSequenceEQLRuleEnabled, isEqlSequenceQuery]); + return true; + }, [isAlertSuppressionForSequenceEQLRuleEnabled, isEqlSequenceQuery]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), From fe47eb51c40d11c502bab388abbe3845dbd76811 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Tue, 3 Dec 2024 20:06:01 -0500 Subject: [PATCH 70/70] fix translation files --- x-pack/plugins/translations/translations/fr-FR.json | 6 ------ x-pack/plugins/translations/translations/ja-JP.json | 6 ------ x-pack/plugins/translations/translations/zh-CN.json | 6 ------ 3 files changed, 18 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 52b0b357720e2..6dfc8c8ec84b4 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -37837,12 +37837,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "Au moins une correspondance d'indicateur est requise.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "Requête EQL", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "Une requête EQL est requise.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "Ouvrir une fenêtre contextuelle d'aide", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "Consultez {createEsqlRuleTypeLink} pour commencer à utiliser les règles ES|QL.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "documentation", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "Requête ES|QL", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "La suppression n'est pas prise en charge pour les requêtes de séquence EQL.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Remplacez la requête EQL par une requête non séquentielle ou supprimez les champs de suppression.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "Seuil de score d'anomalie", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Tâche de Machine Learning", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "Requête personnalisée", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index a39d810d1825e..7bedaa1743684 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -37694,12 +37694,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "1 つ以上のインジケーター一致が必要です。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL クエリ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQLクエリは必須です。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "ヘルプポップオーバーを開く", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "ES|QL ルールの使用を開始するには、{createEsqlRuleTypeLink}を確認してください。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "ドキュメンテーション", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "ES|QLクエリ", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQLシーケンスクエリでは抑制はサポートされていません。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} EQLクエリを非シーケンスクエリに変更するか、抑制フィールドを削除してください。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "異常スコアしきい値", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "機械学習ジョブ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "カスタムクエリー", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a9d9bedc59757..f349cbeaa21e5 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -37121,12 +37121,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "至少需要一个指标匹配。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL 查询", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQL 查询必填。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoAriaLabel": "打开帮助弹出框", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipContent": "请访问我们的{createEsqlRuleTypeLink}以开始使用 ES|QL 规则。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlInfoTooltipLink": "文档", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.esqlQueryLabel": "ES|QL 查询", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQL 序列查询不支持阻止。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} 将 EQL 查询更改为非序列查询，或移除阻止字段。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "异常分数阈值", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Machine Learning 作业", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "定制查询",