diff --git a/packages/kbn-rule-data-utils/index.ts b/packages/kbn-rule-data-utils/index.ts index 897e5609a8347..18c81aba6d81a 100644 --- a/packages/kbn-rule-data-utils/index.ts +++ b/packages/kbn-rule-data-utils/index.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +export * from './src/default_alerts_as_data'; export * from './src/technical_field_names'; export * from './src/alerts_as_data_rbac'; export * from './src/alerts_as_data_severity'; diff --git a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts new file mode 100644 index 0000000000000..b428bea94cdcd --- /dev/null +++ b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ValuesType } from 'utility-types'; + +const KIBANA_NAMESPACE = 'kibana' as const; +const ALERT_NAMESPACE = `${KIBANA_NAMESPACE}.alert` as const; +const ALERT_RULE_NAMESPACE = `${ALERT_NAMESPACE}.rule` as const; + +// kibana.space_ids - space ID(s) of the rule that created this alert +const SPACE_IDS = `${KIBANA_NAMESPACE}.space_ids` as const; + +// kibana.version - Kibana version that this alert was created +const VERSION = `${KIBANA_NAMESPACE}.version` as const; + +// kibana.alert.action_group - framework action group ID for this alert +const ALERT_ACTION_GROUP = `${ALERT_NAMESPACE}.action_group` as const; + +// kibana.alert.duration.us - alert duration in nanoseconds - updated each execution +// that the alert is active +const ALERT_DURATION = `${ALERT_NAMESPACE}.duration.us` as const; + +// kibana.alert.end - timestamp when the alert is auto-recovered by the framework +const ALERT_END = `${ALERT_NAMESPACE}.end` as const; + +// kibana.alert.flapping - whether the alert is currently in a flapping state +const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const; + +// kibana.alert.id - alert ID, also known as alert instance ID +const ALERT_ID = `${ALERT_NAMESPACE}.id` as const; + +// kibana.alert.reason - human readable reason that this alert is active +const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const; + +// kibana.alert.start - timestamp when the alert is first active +const ALERT_START = `${ALERT_NAMESPACE}.start` as const; + +// kibana.alert.status - active/recovered status of alert +const ALERT_STATUS = `${ALERT_NAMESPACE}.status` as const; + +// kibana.alert.time_range - time range of alert from kibana.alert.start to now +const ALERT_TIME_RANGE = `${ALERT_NAMESPACE}.time_range` as const; + +// kibana.alert.uuid - unique ID for the active span of this alert +const ALERT_UUID = `${ALERT_NAMESPACE}.uuid` as const; + +// kibana.alert.workflow_status - open/closed status of alert +const ALERT_WORKFLOW_STATUS = `${ALERT_NAMESPACE}.workflow_status` as const; + +// kibana.alert.rule.category - rule type name for rule that generated this alert +const ALERT_RULE_CATEGORY = `${ALERT_RULE_NAMESPACE}.category` as const; + +// kibana.alert.rule.consumer - consumer for rule that generated this alert +const ALERT_RULE_CONSUMER = `${ALERT_RULE_NAMESPACE}.consumer` as const; + +// kibana.alert.rule.execution.uuid - unique ID for the rule execution that generated this alert +const ALERT_RULE_EXECUTION_UUID = `${ALERT_RULE_NAMESPACE}.execution.uuid` as const; + +// kibana.alert.rule.name - rule name for rule that generated this alert +const ALERT_RULE_NAME = `${ALERT_RULE_NAMESPACE}.name` as const; + +// kibana.alert.rule.parameters - rule parameters for rule that generated this alert +const ALERT_RULE_PARAMETERS = `${ALERT_RULE_NAMESPACE}.parameters` as const; + +// kibana.alert.rule.producer - rule type producer for rule that generated this alert +const ALERT_RULE_PRODUCER = `${ALERT_RULE_NAMESPACE}.producer` as const; + +// kibana.alert.rule.tags - rule tags for rule that generated this alert +const ALERT_RULE_TAGS = `${ALERT_RULE_NAMESPACE}.tags` as const; + +// kibana.alert.rule_type_id - rule type id for rule that generated this alert +const ALERT_RULE_TYPE_ID = `${ALERT_RULE_NAMESPACE}.rule_type_id` as const; + +// kibana.alert.rule.uuid - rule ID for rule that generated this alert +const ALERT_RULE_UUID = `${ALERT_RULE_NAMESPACE}.uuid` as const; + +const namespaces = { + KIBANA_NAMESPACE, + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, +}; + +const fields = { + ALERT_ACTION_GROUP, + ALERT_DURATION, + ALERT_END, + ALERT_FLAPPING, + ALERT_ID, + ALERT_REASON, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_NAME, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_TAGS, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UUID, + ALERT_START, + ALERT_STATUS, + ALERT_TIME_RANGE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, + VERSION, +}; + +export { + ALERT_ACTION_GROUP, + ALERT_DURATION, + ALERT_END, + ALERT_FLAPPING, + ALERT_ID, + ALERT_REASON, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_NAME, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_TAGS, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UUID, + ALERT_START, + ALERT_STATUS, + ALERT_TIME_RANGE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, + VERSION, + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, + KIBANA_NAMESPACE, +}; + +export type DefaultAlertFieldName = ValuesType; diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index b5c4516c6c2c7..89eca0f923046 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -7,41 +7,51 @@ */ import { ValuesType } from 'utility-types'; +import { + KIBANA_NAMESPACE, + ALERT_ACTION_GROUP, + ALERT_DURATION, + ALERT_END, + ALERT_FLAPPING, + ALERT_REASON, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_NAME, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_TAGS, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UUID, + ALERT_START, + ALERT_STATUS, + ALERT_TIME_RANGE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, + VERSION, + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, +} from './default_alerts_as_data'; -const KIBANA_NAMESPACE = 'kibana' as const; - -const ALERT_NAMESPACE = `${KIBANA_NAMESPACE}.alert` as const; -const ALERT_RULE_NAMESPACE = `${ALERT_NAMESPACE}.rule` as const; const ALERT_RULE_THREAT_NAMESPACE = `${ALERT_RULE_NAMESPACE}.threat` as const; const ECS_VERSION = 'ecs.version' as const; const EVENT_ACTION = 'event.action' as const; const EVENT_KIND = 'event.kind' as const; const EVENT_MODULE = 'event.module' as const; -const SPACE_IDS = `${KIBANA_NAMESPACE}.space_ids` as const; const TAGS = 'tags' as const; const TIMESTAMP = '@timestamp' as const; -const VERSION = `${KIBANA_NAMESPACE}.version` as const; // Fields pertaining to the alert -const ALERT_ACTION_GROUP = `${ALERT_NAMESPACE}.action_group` as const; const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const; -const ALERT_DURATION = `${ALERT_NAMESPACE}.duration.us` as const; -const ALERT_END = `${ALERT_NAMESPACE}.end` as const; const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const; const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const; -const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const; const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; -const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const; const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const; const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const; -const ALERT_START = `${ALERT_NAMESPACE}.start` as const; -const ALERT_TIME_RANGE = `${ALERT_NAMESPACE}.time_range` as const; -const ALERT_STATUS = `${ALERT_NAMESPACE}.status` as const; const ALERT_SYSTEM_STATUS = `${ALERT_NAMESPACE}.system_status` as const; -const ALERT_UUID = `${ALERT_NAMESPACE}.uuid` as const; const ALERT_WORKFLOW_REASON = `${ALERT_NAMESPACE}.workflow_reason` as const; -const ALERT_WORKFLOW_STATUS = `${ALERT_NAMESPACE}.workflow_status` as const; const ALERT_WORKFLOW_USER = `${ALERT_NAMESPACE}.workflow_user` as const; const ALERT_SUPPRESSION_META = `${ALERT_NAMESPACE}.suppression` as const; const ALERT_SUPPRESSION_TERMS = `${ALERT_SUPPRESSION_META}.terms` as const; @@ -61,22 +71,16 @@ const ALERT_RULE_CREATED_BY = `${ALERT_RULE_NAMESPACE}.created_by` as const; const ALERT_RULE_DESCRIPTION = `${ALERT_RULE_NAMESPACE}.description` as const; const ALERT_RULE_ENABLED = `${ALERT_RULE_NAMESPACE}.enabled` as const; const ALERT_RULE_EXCEPTIONS_LIST = `${ALERT_RULE_NAMESPACE}.exceptions_list` as const; -const ALERT_RULE_EXECUTION_UUID = `${ALERT_RULE_NAMESPACE}.execution.uuid` as const; const ALERT_RULE_FROM = `${ALERT_RULE_NAMESPACE}.from` as const; const ALERT_RULE_INTERVAL = `${ALERT_RULE_NAMESPACE}.interval` as const; const ALERT_RULE_LICENSE = `${ALERT_RULE_NAMESPACE}.license` as const; -const ALERT_RULE_CATEGORY = `${ALERT_RULE_NAMESPACE}.category` as const; -const ALERT_RULE_NAME = `${ALERT_RULE_NAMESPACE}.name` as const; const ALERT_RULE_NAMESPACE_FIELD = `${ALERT_RULE_NAMESPACE}.namespace` as const; const ALERT_RULE_NOTE = `${ALERT_RULE_NAMESPACE}.note` as const; -const ALERT_RULE_PARAMETERS = `${ALERT_RULE_NAMESPACE}.parameters` as const; const ALERT_RULE_REFERENCES = `${ALERT_RULE_NAMESPACE}.references` as const; const ALERT_RULE_RULE_ID = `${ALERT_RULE_NAMESPACE}.rule_id` as const; const ALERT_RULE_RULE_NAME_OVERRIDE = `${ALERT_RULE_NAMESPACE}.rule_name_override` as const; -const ALERT_RULE_TAGS = `${ALERT_RULE_NAMESPACE}.tags` as const; const ALERT_RULE_TO = `${ALERT_RULE_NAMESPACE}.to` as const; const ALERT_RULE_TYPE = `${ALERT_RULE_NAMESPACE}.type` as const; -const ALERT_RULE_TYPE_ID = `${ALERT_RULE_NAMESPACE}.rule_type_id` as const; const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const; const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const; const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const; @@ -97,16 +101,6 @@ const ALERT_THREAT_TECHNIQUE_SUBTECHNIQUE_NAME = const ALERT_THREAT_TECHNIQUE_SUBTECHNIQUE_REFERENCE = `${ALERT_RULE_THREAT_NAMESPACE}.technique.subtechnique.reference` as const; -// the feature instantiating a rule type. -// Rule created in stack --> alerts -// Rule created in siem --> siem -const ALERT_RULE_CONSUMER = `${ALERT_RULE_NAMESPACE}.consumer` as const; -// the plugin that registered the rule type. -// Rule type apm.error_rate --> apm -// Rule type siem.signals --> siem -const ALERT_RULE_PRODUCER = `${ALERT_RULE_NAMESPACE}.producer` as const; -const ALERT_RULE_UUID = `${ALERT_RULE_NAMESPACE}.uuid` as const; - const namespaces = { KIBANA_NAMESPACE, ALERT_NAMESPACE, @@ -189,23 +183,12 @@ const fields = { }; export { - ALERT_ACTION_GROUP, ALERT_BUILDING_BLOCK_TYPE, - ALERT_DURATION, - ALERT_END, ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE, - ALERT_FLAPPING, ALERT_INSTANCE_ID, - ALERT_NAMESPACE, - ALERT_RULE_NAMESPACE, - ALERT_RULE_CONSUMER, - ALERT_RULE_PRODUCER, - ALERT_REASON, ALERT_RISK_SCORE, - ALERT_STATUS, ALERT_WORKFLOW_REASON, - ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_USER, ALERT_CASE_IDS, ALERT_RULE_AUTHOR, @@ -214,36 +197,25 @@ export { ALERT_RULE_DESCRIPTION, ALERT_RULE_ENABLED, ALERT_RULE_EXCEPTIONS_LIST, - ALERT_RULE_EXECUTION_UUID, ALERT_RULE_FROM, ALERT_RULE_INTERVAL, ALERT_RULE_LICENSE, - ALERT_RULE_NAME, ALERT_RULE_NAMESPACE_FIELD, ALERT_RULE_NOTE, - ALERT_RULE_PARAMETERS, ALERT_RULE_REFERENCES, ALERT_RULE_RULE_ID, ALERT_RULE_RULE_NAME_OVERRIDE, - ALERT_RULE_TAGS, ALERT_RULE_TO, ALERT_RULE_TYPE, - ALERT_RULE_TYPE_ID, ALERT_RULE_UPDATED_AT, ALERT_RULE_UPDATED_BY, ALERT_RULE_VERSION, ALERT_SEVERITY, - ALERT_START, - ALERT_TIME_RANGE, ALERT_SYSTEM_STATUS, - ALERT_UUID, ECS_VERSION, EVENT_ACTION, EVENT_KIND, EVENT_MODULE, - KIBANA_NAMESPACE, - ALERT_RULE_UUID, - ALERT_RULE_CATEGORY, ALERT_THREAT_FRAMEWORK, ALERT_THREAT_TACTIC_ID, ALERT_THREAT_TACTIC_NAME, @@ -262,8 +234,6 @@ export { ALERT_SUPPRESSION_DOCS_COUNT, TAGS, TIMESTAMP, - SPACE_IDS, - VERSION, }; export type TechnicalRuleDataFieldName = ValuesType; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts new file mode 100644 index 0000000000000..4613415e0fa00 --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_ACTION_GROUP, + ALERT_DURATION, + ALERT_END, + ALERT_FLAPPING, + ALERT_ID, + ALERT_REASON, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_NAME, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_TAGS, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UUID, + ALERT_START, + ALERT_STATUS, + ALERT_TIME_RANGE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, + VERSION, +} from '@kbn/rule-data-utils'; + +export const alertFieldMap = { + [ALERT_RULE_PARAMETERS]: { + type: 'object', + enabled: false, + required: false, + }, + [ALERT_RULE_TYPE_ID]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_RULE_CONSUMER]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_RULE_PRODUCER]: { + type: 'keyword', + array: false, + required: true, + }, + [SPACE_IDS]: { + type: 'keyword', + array: true, + required: true, + }, + [ALERT_UUID]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_ID]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_START]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_TIME_RANGE]: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + array: false, + required: false, + }, + [ALERT_END]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_DURATION]: { + type: 'long', + array: false, + required: false, + }, + [ALERT_STATUS]: { + type: 'keyword', + array: false, + required: true, + }, + [VERSION]: { + type: 'version', + array: false, + required: false, + }, + [ALERT_WORKFLOW_STATUS]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_ACTION_GROUP]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_REASON]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_CATEGORY]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_RULE_UUID]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_RULE_EXECUTION_UUID]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_NAME]: { + type: 'keyword', + array: false, + required: true, + }, + [ALERT_RULE_TAGS]: { + type: 'keyword', + array: true, + required: false, + }, + [ALERT_FLAPPING]: { + type: 'boolean', + array: false, + required: false, + }, +}; + +export type AlertFieldMap = typeof alertFieldMap; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts new file mode 100644 index 0000000000000..b4cd25a4f4126 --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { mappingFromFieldMap } from './mapping_from_field_map'; +import { FieldMap } from './types'; + +export interface GetComponentTemplateFromFieldMapOpts { + name: string; + fieldLimit?: number; + fieldMap: FieldMap; +} +export const getComponentTemplateFromFieldMap = ({ + name, + fieldMap, + fieldLimit, +}: GetComponentTemplateFromFieldMapOpts): ClusterPutComponentTemplateRequest => { + return { + name, + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': fieldLimit ?? 1000, + }, + mappings: mappingFromFieldMap(fieldMap, 'strict'), + }, + }; +}; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts new file mode 100644 index 0000000000000..2f2cac2367e8b --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -0,0 +1,282 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { mappingFromFieldMap } from './mapping_from_field_map'; +import { FieldMap } from './types'; +import { alertFieldMap } from './alert_field_map'; + +describe('mappingFromFieldMap', () => { + const fieldMap: FieldMap = { + date_field: { + type: 'date', + array: false, + required: true, + }, + keyword_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + long_field: { + type: 'long', + array: false, + required: false, + }, + multifield_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + multi_fields: [ + { + flat_name: 'multifield_field.text', + name: 'text', + type: 'match_only_text', + }, + ], + }, + geopoint_field: { + type: 'geo_point', + array: false, + required: false, + }, + ip_field: { + type: 'ip', + array: false, + required: false, + }, + array_field: { + type: 'keyword', + array: true, + required: false, + ignore_above: 1024, + }, + nested_array_field: { + type: 'nested', + array: false, + required: false, + }, + 'nested_array_field.field1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'nested_array_field.field2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + scaled_float_field: { + type: 'scaled_float', + array: false, + required: false, + scaling_factor: 1000, + }, + constant_keyword_field: { + type: 'constant_keyword', + array: false, + required: false, + }, + 'parent_field.child1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'parent_field.child2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + unmapped_object: { + type: 'object', + required: false, + enabled: false, + }, + formatted_field: { + type: 'date_range', + required: false, + format: 'epoch_millis||strict_date_optional_time', + }, + }; + const expectedMapping = { + properties: { + array_field: { + ignore_above: 1024, + type: 'keyword', + }, + constant_keyword_field: { + type: 'constant_keyword', + }, + date_field: { + type: 'date', + }, + geopoint_field: { + type: 'geo_point', + }, + ip_field: { + type: 'ip', + }, + keyword_field: { + ignore_above: 1024, + type: 'keyword', + }, + long_field: { + type: 'long', + }, + multifield_field: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + nested_array_field: { + properties: { + field1: { + ignore_above: 1024, + type: 'keyword', + }, + field2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + parent_field: { + properties: { + child1: { + ignore_above: 1024, + type: 'keyword', + }, + child2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + scaled_float_field: { + scaling_factor: 1000, + type: 'scaled_float', + }, + unmapped_object: { + enabled: false, + type: 'object', + }, + formatted_field: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + }, + }; + it('correctly creates mapping from field map', () => { + expect(mappingFromFieldMap(fieldMap)).toEqual({ dynamic: 'strict', ...expectedMapping }); + expect(mappingFromFieldMap(alertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + kibana: { + properties: { + alert: { + properties: { + action_group: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + end: { + type: 'date', + }, + flapping: { + type: 'boolean', + }, + id: { + type: 'keyword', + }, + reason: { + type: 'keyword', + }, + rule: { + properties: { + category: { + type: 'keyword', + }, + consumer: { + type: 'keyword', + }, + execution: { + properties: { + uuid: { + type: 'keyword', + }, + }, + }, + name: { + type: 'keyword', + }, + parameters: { + type: 'object', + enabled: false, + }, + producer: { + type: 'keyword', + }, + rule_type_id: { + type: 'keyword', + }, + tags: { + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + }, + }, + start: { + type: 'date', + }, + status: { + type: 'keyword', + }, + time_range: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + uuid: { + type: 'keyword', + }, + workflow_status: { + type: 'keyword', + }, + }, + }, + space_ids: { + type: 'keyword', + }, + version: { + type: 'version', + }, + }, + }, + }, + }); + }); + + it('uses dynamic setting if specified', () => { + expect(mappingFromFieldMap(fieldMap, true)).toEqual({ dynamic: true, ...expectedMapping }); + }); +}); diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts new file mode 100644 index 0000000000000..5a1de7a995b36 --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { set } from '@kbn/safer-lodash-set'; +import { FieldMap, MultiField } from './types'; + +export function mappingFromFieldMap( + fieldMap: FieldMap, + dynamic: 'strict' | boolean = 'strict' +): MappingTypeMapping { + const mappings = { + dynamic, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key: string) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { name, required, array, multi_fields, ...rest } = field; + + const mapped = multi_fields + ? { + ...rest, + // eslint-disable-next-line @typescript-eslint/naming-convention + fields: multi_fields.reduce((acc, multi_field: MultiField) => { + return { + ...acc, + [multi_field.name]: { + type: multi_field.type, + }, + }; + }, {}), + } + : rest; + + set(mappings.properties, field.name.split('.').join('.properties.'), mapped); + }); + + return mappings; +} diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts new file mode 100644 index 0000000000000..b687cbfb0cf7d --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface MultiField { + flat_name?: string; + name: string; + type: string; +} + +export interface FieldMap { + [key: string]: { + type: string; + required: boolean; + array?: boolean; + doc_values?: boolean; + enabled?: boolean; + format?: string; + ignore_above?: number; + index?: boolean; + multi_fields?: MultiField[]; + path?: string; + scaling_factor?: number; + dynamic?: boolean | string; + }; +} diff --git a/x-pack/plugins/alerting/common/alert_schema/index.ts b/x-pack/plugins/alerting/common/alert_schema/index.ts new file mode 100644 index 0000000000000..acca43450fe34 --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { alertFieldMap } from './field_maps/alert_field_map'; +export { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts new file mode 100644 index 0000000000000..d11e95f909c19 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const creatAlertsServiceMock = () => { + return jest.fn().mockImplementation(() => { + return { + initialize: jest.fn(), + register: jest.fn(), + isInitialized: jest.fn(), + isContextInitialized: jest.fn(), + }; + }); +}; + +export const alertsServiceMock = { + create: creatAlertsServiceMock(), +}; diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts new file mode 100644 index 0000000000000..ba3623526591f --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts @@ -0,0 +1,944 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { ReplaySubject, Subject } from 'rxjs'; +import { AlertsService } from './alerts_service'; + +let logger: ReturnType; +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const SimulateTemplateResponse = { + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, +}; +interface HTTPError extends Error { + statusCode: number; +} + +interface EsError extends Error { + meta: { + body: { + error: { + type: string; + }; + }; + }; +} + +const GetAliasResponse = { + real_index: { + aliases: { + alias_1: { + is_hidden: true, + }, + alias_2: { + is_hidden: true, + }, + }, + }, +}; + +const IlmPutBody = { + body: { + policy: { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }, + }, + name: 'alerts-default-ilm-policy', +}; + +const getIndexTemplatePutBody = (context?: string) => ({ + name: `.alerts-${context ? context : 'test'}-default-template`, + body: { + index_patterns: [`.alerts-${context ? context : 'test'}-default-*`], + composed_of: [ + 'alerts-common-component-template', + `alerts-${context ? context : 'test'}-component-template`, + ], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: 'alerts-default-ilm-policy', + rollover_alias: `.alerts-${context ? context : 'test'}-default`, + }, + 'index.mapping.total_fields.limit': 2500, + }, + mappings: { + dynamic: false, + }, + }, + _meta: { + managed: true, + }, + }, +}); + +const TestRegistrationContext = { + context: 'test', + fieldMap: { field: { type: 'keyword', required: false } }, +}; + +const AnotherRegistrationContext = { + context: 'another', + fieldMap: { field: { type: 'keyword', required: false } }, +}; + +describe('Alerts Service', () => { + let pluginStop$: Subject; + + beforeEach(() => { + jest.clearAllMocks(); + logger = loggingSystemMock.createLogger(); + pluginStop$ = new ReplaySubject(1); + jest.spyOn(global.Math, 'random').mockReturnValue(0.01); + clusterClient.indices.simulateTemplate.mockImplementation(async () => SimulateTemplateResponse); + clusterClient.indices.simulateIndexTemplate.mockImplementation( + async () => SimulateTemplateResponse + ); + clusterClient.indices.getAlias.mockImplementation(async () => GetAliasResponse); + }); + + afterEach(() => { + pluginStop$.next(); + pluginStop$.complete(); + }); + describe('initialize()', () => { + test('should correctly initialize common resources', async () => { + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(alertsService.isInitialized()).toEqual(true); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + }); + + test('should log error and set initialized to false if adding ILM policy throws error', async () => { + clusterClient.ilm.putLifecycle.mockRejectedValueOnce(new Error('fail')); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(alertsService.isInitialized()).toEqual(false); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing ILM policy alerts-default-ilm-policy - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).not.toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if creating/updating common component template throws error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(new Error('fail')); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(alertsService.isInitialized()).toEqual(false); + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template alerts-common-component-template - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + }); + + test('should install resources for contexts awaiting initialization when common resources are initialized', async () => { + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + // pre-register contexts so they get installed right after initialization + alertsService.register(TestRegistrationContext); + alertsService.register(AnotherRegistrationContext); + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(alertsService.isInitialized()).toEqual(true); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + expect(await alertsService.isContextInitialized(AnotherRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + // 1x for common component template, 2x for context specific + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('alerts-another-component-template'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('alerts-test-component-template'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( + 1, + getIndexTemplatePutBody('another') + ); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( + 2, + getIndexTemplatePutBody() + ); + + expect(clusterClient.indices.getAlias).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(1, { + index: '.alerts-another-default-*', + }); + expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(2, { + index: '.alerts-test-default-*', + }); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(4); + expect(clusterClient.indices.create).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenNthCalledWith(1, { + index: '.alerts-another-default-000001', + body: { + aliases: { + '.alerts-another-default': { + is_write_index: true, + }, + }, + }, + }); + expect(clusterClient.indices.create).toHaveBeenNthCalledWith(2, { + index: '.alerts-test-default-000001', + body: { + aliases: { + '.alerts-test-default': { + is_write_index: true, + }, + }, + }, + }); + }); + }); + + describe('register()', () => { + let alertsService: AlertsService; + beforeEach(async () => { + alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + expect(alertsService.isInitialized()).toEqual(true); + }); + + test('should correctly install resources for context when common initialization is complete', async () => { + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('alerts-test-component-template'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + getIndexTemplatePutBody() + ); + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.alerts-test-default-*', + }); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.alerts-test-default-000001', + body: { + aliases: { + '.alerts-test-default': { + is_write_index: true, + }, + }, + }, + }); + }); + + test('should not install component template for context fieldMap is empty', async () => { + alertsService.register({ + context: 'empty', + fieldMap: {}, + }); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized('empty')).toEqual(true); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: `.alerts-empty-default-template`, + body: { + index_patterns: [`.alerts-empty-default-*`], + composed_of: ['alerts-common-component-template'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: 'alerts-default-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 2500, + }, + mappings: { + dynamic: false, + }, + }, + _meta: { + managed: true, + }, + }, + }); + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.alerts-empty-default-*', + }); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.alerts-empty-default-000001', + body: { + aliases: { + '.alerts-empty-default': { + is_write_index: true, + }, + }, + }, + }); + }); + + test('should skip initialization if context already exists', async () => { + alertsService.register(TestRegistrationContext); + alertsService.register(TestRegistrationContext); + + expect(logger.debug).toHaveBeenCalledWith( + `Resources for context "test" have already been registered.` + ); + }); + + test('should throw error if context already exists and has been registered with a different field map', async () => { + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(() => { + alertsService.register({ + ...TestRegistrationContext, + fieldMap: { anotherField: { type: 'keyword', required: false } }, + }); + }).toThrowErrorMatchingInlineSnapshot( + `"test has already been registered with a different mapping"` + ); + }); + + test('should not update index template if simulating template throws error', async () => { + clusterClient.indices.simulateTemplate.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(logger.error).toHaveBeenCalledWith( + `Failed to simulate index template mappings for .alerts-test-default-template; not applying mappings - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + // putIndexTemplate is skipped but other operations are called as expected + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if simulating template returns empty mappings', async () => { + clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...SimulateTemplateResponse, + template: { + ...SimulateTemplateResponse.template, + mappings: {}, + }, + })); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith( + new Error( + `No mappings would be generated for .alerts-test-default-template, possibly due to failed/misconfigured bootstrapping` + ) + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if updating index template throws error', async () => { + clusterClient.indices.putIndexTemplate.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test-default-template - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if checking for concrete write index throws error', async () => { + clusterClient.indices.getAlias.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith( + `Error fetching concrete indices for .alerts-test-default-* pattern - fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('should not throw error if checking for concrete write index throws 404', async () => { + const error = new Error(`index doesn't exist`) as HTTPError; + error.statusCode = 404; + clusterClient.indices.getAlias.mockRejectedValueOnce(error); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if updating index settings for existing indices throws error', async () => { + clusterClient.indices.putSettings.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith( + `Failed to PUT index.mapping.total_fields.limit settings for alias alias_1: fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('should skip updating index mapping for existing indices if simulate index template throws error', async () => { + clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(logger.error).toHaveBeenCalledWith( + `Ignored PUT mappings for alias alias_1; error generating simulated mappings: fail` + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if updating index mappings for existing indices throws error', async () => { + clusterClient.indices.putMapping.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith(`Failed to PUT mapping for alias alias_1: fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('does not updating settings or mappings if no existing concrete indices', async () => { + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({})); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).not.toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if concrete indices exist but none are write index', async () => { + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ + '.alerts-test-default-0001': { + aliases: { + '.alerts-test-default': { + is_write_index: false, + is_hidden: true, + }, + alias_2: { + is_write_index: false, + is_hidden: true, + }, + }, + }, + })); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith( + new Error( + `Indices matching pattern .alerts-test-default-* exist but none are set as the write index for alias .alerts-test-default` + ) + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('does not create new index if concrete write index exists', async () => { + clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ + '.alerts-test-default-0001': { + aliases: { + '.alerts-test-default': { + is_write_index: true, + is_hidden: true, + }, + alias_2: { + is_write_index: false, + is_hidden: true, + }, + }, + }, + })); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).not.toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if create concrete index throws error', async () => { + clusterClient.indices.create.mockRejectedValueOnce(new Error('fail')); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should not throw error if create concrete index throws resource_already_exists_exception error and write index already exists', async () => { + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.alerts-test-default-000001': { + aliases: { '.alerts-test-default': { is_write_index: true } }, + }, + })); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.get).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + + test('should log error and set initialized to false if create concrete index throws resource_already_exists_exception error and write index does not already exists', async () => { + const error = new Error(`fail`) as EsError; + error.meta = { + body: { + error: { + type: 'resource_already_exists_exception', + }, + }, + }; + clusterClient.indices.create.mockRejectedValueOnce(error); + clusterClient.indices.get.mockImplementationOnce(async () => ({ + '.alerts-test-default-000001': { + aliases: { '.alerts-test-default': { is_write_index: false } }, + }, + })); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.getAlias).toHaveBeenCalled(); + expect(clusterClient.indices.putSettings).toHaveBeenCalled(); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalled(); + expect(clusterClient.indices.putMapping).toHaveBeenCalled(); + expect(clusterClient.indices.get).toHaveBeenCalled(); + expect(clusterClient.indices.create).toHaveBeenCalled(); + }); + }); + + describe('retries', () => { + test('should retry adding ILM policy for transient ES errors', async () => { + clusterClient.ilm.putLifecycle + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(3); + }); + + test('should retry adding component template for transient ES errors', async () => { + clusterClient.cluster.putComponentTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index template for transient ES errors', async () => { + clusterClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 150)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); + + test('should retry updating index settings for existing indices for transient ES errors', async () => { + clusterClient.indices.putSettings + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 150)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); + }); + + test('should retry updating index mappings for existing indices for transient ES errors', async () => { + clusterClient.indices.putMapping + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 150)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(4); + }); + + test('should retry creating concrete index for transient ES errors', async () => { + clusterClient.indices.create + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ index: 'index', shards_acknowledged: true, acknowledged: true }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(true); + + alertsService.register(TestRegistrationContext); + await new Promise((r) => setTimeout(r, 150)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); + }); + }); + + describe('timeout', () => { + test('should short circuit initialization if timeout exceeded', async () => { + clusterClient.ilm.putLifecycle.mockImplementationOnce(async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return { acknowledged: true }; + }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(10); + await new Promise((r) => setTimeout(r, 150)); + expect(alertsService.isInitialized()).toEqual(false); + + expect(logger.error).toHaveBeenCalledWith(new Error(`Timeout: it took more than 10ms`)); + }); + + test('should short circuit initialization if pluginStop$ signal received but not throw error', async () => { + pluginStop$.next(); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(logger.error).toHaveBeenCalledWith( + new Error(`Server is stopping; must stop all async operations`) + ); + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts new file mode 100644 index 0000000000000..0742be50c4fa4 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts @@ -0,0 +1,512 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ClusterPutComponentTemplateRequest, + IndicesSimulateIndexTemplateResponse, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/types'; +import { get, isEmpty, isEqual } from 'lodash'; +import { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { firstValueFrom, Observable } from 'rxjs'; +import { FieldMap } from '../../common/alert_schema/field_maps/types'; +import { alertFieldMap } from '../../common/alert_schema'; +import { ILM_POLICY_NAME, DEFAULT_ILM_POLICY } from './default_lifecycle_policy'; +import { + getComponentTemplate, + getComponentTemplateName, + getIndexTemplateAndPattern, + IIndexPatternString, +} from './types'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; +import { IRuleTypeAlerts } from '../types'; +import { + createResourceInstallationHelper, + ResourceInstallationHelper, +} from './create_resource_installation_helper'; + +const TOTAL_FIELDS_LIMIT = 2500; +const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes + +interface AlertsServiceParams { + logger: Logger; + pluginStop$: Observable; + elasticsearchClientPromise: Promise; +} + +interface ConcreteIndexInfo { + index: string; + alias: string; + isWriteIndex: boolean; +} +interface IAlertsService { + /** + * Initializes the common ES resources needed for framework alerts as data + * - ILM policy - common policy shared by all AAD indices + * - Component template - common mappings for fields populated and used by the framework + * + * Once common resource initialization is complete, look for any solution-specific + * resources that have been registered and are awaiting initialization. + */ + initialize(timeoutMs?: number): void; + + /** + * Register solution specific resources. If common resource initialization is + * complete, go ahead and install those resources, otherwise add to queue to + * await initialization + * + * Solution specific resources include: + * - Component template - solution specific mappings for fields used only by solution rule types + * - Index templates - solution specific template that combines common and solution specific component templates + * - Concrete write index - solution specific write index + */ + register(opts: IRuleTypeAlerts, timeoutMs?: number): void; + + isInitialized(): boolean; +} + +export class AlertsService implements IAlertsService { + private initialized: boolean; + private resourceInitializationHelper: ResourceInstallationHelper; + private registeredContexts: Map = new Map(); + + constructor(private readonly options: AlertsServiceParams) { + this.initialized = false; + this.resourceInitializationHelper = createResourceInstallationHelper( + this.initializeContext.bind(this) + ); + } + + public isInitialized() { + return this.initialized; + } + + public async isContextInitialized(context: string) { + return (await this.resourceInitializationHelper.getInitializedContexts().get(context)) ?? false; + } + + public initialize(timeoutMs?: number) { + // Only initialize once + if (this.initialized) return; + + this.options.logger.debug(`Initializing resources for AlertsService`); + + // Use setImmediate to execute async fns as soon as possible + setImmediate(async () => { + try { + const esClient = await this.options.elasticsearchClientPromise; + + // Common initialization installs ILM policy and shared component template + const initFns = [ + () => this.createOrUpdateIlmPolicy(esClient), + () => this.createOrUpdateComponentTemplate(esClient, getComponentTemplate(alertFieldMap)), + ]; + + for (const fn of initFns) { + await this.installWithTimeout(async () => await fn(), timeoutMs); + } + + this.initialized = true; + } catch (err) { + this.options.logger.error( + `Error installing common resources for AlertsService. No additional resources will be installed and rule execution may be impacted.` + ); + this.initialized = false; + } + + if (this.initialized) { + this.resourceInitializationHelper.setReadyToInitialize(timeoutMs); + } + }); + } + + public register({ context, fieldMap }: IRuleTypeAlerts, timeoutMs?: number) { + // check whether this context has been registered before + if (this.registeredContexts.has(context)) { + const registeredFieldMap = this.registeredContexts.get(context); + if (!isEqual(fieldMap, registeredFieldMap)) { + throw new Error(`${context} has already been registered with a different mapping`); + } + this.options.logger.debug(`Resources for context "${context}" have already been registered.`); + return; + } + + this.options.logger.info(`Registering resources for context "${context}".`); + this.registeredContexts.set(context, fieldMap); + this.resourceInitializationHelper.add({ context, fieldMap }, timeoutMs); + } + + private async initializeContext({ context, fieldMap }: IRuleTypeAlerts, timeoutMs?: number) { + const esClient = await this.options.elasticsearchClientPromise; + + const indexTemplateAndPattern = getIndexTemplateAndPattern(context); + + // Context specific initialization installs component template, index template and write index + // If fieldMap is empty, don't create context specific component template + const initFns = isEmpty(fieldMap) + ? [ + async () => + await this.createOrUpdateIndexTemplate(esClient, indexTemplateAndPattern, [ + getComponentTemplateName(), + ]), + async () => await this.createConcreteWriteIndex(esClient, indexTemplateAndPattern), + ] + : [ + async () => + await this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate(fieldMap, context) + ), + async () => + await this.createOrUpdateIndexTemplate(esClient, indexTemplateAndPattern, [ + getComponentTemplateName(), + getComponentTemplateName(context), + ]), + async () => await this.createConcreteWriteIndex(esClient, indexTemplateAndPattern), + ]; + + for (const fn of initFns) { + await this.installWithTimeout(async () => await fn(), timeoutMs); + } + } + + /** + * Creates ILM policy if it doesn't already exist, updates it if it does + */ + private async createOrUpdateIlmPolicy(esClient: ElasticsearchClient) { + this.options.logger.info(`Installing ILM policy ${ILM_POLICY_NAME}`); + + try { + await retryTransientEsErrors( + () => + esClient.ilm.putLifecycle({ + name: ILM_POLICY_NAME, + body: DEFAULT_ILM_POLICY, + }), + { logger: this.options.logger } + ); + } catch (err) { + this.options.logger.error(`Error installing ILM policy ${ILM_POLICY_NAME} - ${err.message}`); + throw err; + } + } + + private async createOrUpdateComponentTemplate( + esClient: ElasticsearchClient, + template: ClusterPutComponentTemplateRequest + ) { + this.options.logger.info(`Installing component template ${template.name}`); + + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { + logger: this.options.logger, + }); + } catch (err) { + this.options.logger.error( + `Error installing component template ${template.name} - ${err.message}` + ); + throw err; + } + } + + /** + * Installs index template that uses installed component template + * Prior to installation, simulates the installation to check for possible + * conflicts. Simulate should return an empty mapping if a template + * conflicts with an already installed template. + */ + private async createOrUpdateIndexTemplate( + esClient: ElasticsearchClient, + indexPatterns: IIndexPatternString, + componentTemplateNames: string[] + ) { + this.options.logger.info(`Installing index template ${indexPatterns.template}`); + + const indexTemplate = { + name: indexPatterns.template, + body: { + index_patterns: [indexPatterns.pattern], + composed_of: componentTemplateNames, + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: ILM_POLICY_NAME, + rollover_alias: indexPatterns.alias, + }, + 'index.mapping.total_fields.limit': TOTAL_FIELDS_LIMIT, + }, + mappings: { + dynamic: false, + }, + }, + _meta: { + managed: true, + }, + // do we need metadata? like kibana version? doesn't that get updated every version? or just the first version its installed + }, + }; + + let mappings: MappingTypeMapping = {}; + try { + // Simulate the index template to proactively identify any issues with the mapping + const simulateResponse = await esClient.indices.simulateTemplate(indexTemplate); + mappings = simulateResponse.template.mappings; + } catch (err) { + this.options.logger.error( + `Failed to simulate index template mappings for ${indexPatterns.template}; not applying mappings - ${err.message}` + ); + return; + } + + if (isEmpty(mappings)) { + throw new Error( + `No mappings would be generated for ${indexPatterns.template}, possibly due to failed/misconfigured bootstrapping` + ); + } + + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(indexTemplate), { + logger: this.options.logger, + }); + } catch (err) { + this.options.logger.error( + `Error installing index template ${indexPatterns.template} - ${err.message}` + ); + throw err; + } + } + + /** + * Updates the underlying mapping for any existing concrete indices + */ + private async updateIndexMappings( + esClient: ElasticsearchClient, + concreteIndices: ConcreteIndexInfo[] + ) { + this.options.logger.debug( + `Updating underlying mappings for ${concreteIndices.length} indices.` + ); + + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + concreteIndices.map((index) => this.updateTotalFieldLimitSetting(esClient, index)) + ); + + // Update mappings of the found indices. + await Promise.all( + concreteIndices.map((index) => this.updateUnderlyingMapping(esClient, index)) + ); + } + + private async updateTotalFieldLimitSetting( + esClient: ElasticsearchClient, + { index, alias }: ConcreteIndexInfo + ) { + try { + await retryTransientEsErrors( + () => + esClient.indices.putSettings({ + index, + body: { + 'index.mapping.total_fields.limit': TOTAL_FIELDS_LIMIT, + }, + }), + { + logger: this.options.logger, + } + ); + return; + } catch (err) { + this.options.logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for alias ${alias}: ${err.message}` + ); + throw err; + } + } + + private async updateUnderlyingMapping( + esClient: ElasticsearchClient, + { index, alias }: ConcreteIndexInfo + ) { + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await esClient.indices.simulateIndexTemplate({ + name: index, + }); + } catch (err) { + this.options.logger.error( + `Ignored PUT mappings for alias ${alias}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + this.options.logger.error( + `Ignored PUT mappings for alias ${alias}; simulated mappings were empty` + ); + return; + } + + try { + await retryTransientEsErrors( + () => + esClient.indices.putMapping({ + index, + body: simulatedMapping, + }), + { + logger: this.options.logger, + } + ); + + return; + } catch (err) { + this.options.logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); + throw err; + } + } + + private async createConcreteWriteIndex( + esClient: ElasticsearchClient, + indexPatterns: IIndexPatternString + ) { + this.options.logger.info(`Creating concrete write index - ${indexPatterns.name}`); + + // check if a concrete write index already exists + let concreteIndices: ConcreteIndexInfo[] = []; + try { + const response = await esClient.indices.getAlias({ + index: indexPatterns.pattern, + }); + + concreteIndices = Object.entries(response).flatMap(([index, { aliases }]) => + Object.entries(aliases).map(([aliasName, aliasProperties]) => ({ + index, + alias: aliasName, + isWriteIndex: aliasProperties.is_write_index ?? false, + })) + ); + + this.options.logger.debug( + `Found ${concreteIndices.length} concrete indices for ${ + indexPatterns.name + } - ${JSON.stringify(concreteIndices)}` + ); + } catch (error) { + // 404 is expected if no concrete write indices have been created + if (error.statusCode !== 404) { + this.options.logger.error( + `Error fetching concrete indices for ${indexPatterns.pattern} pattern - ${error.message}` + ); + throw error; + } + } + + let concreteWriteIndicesExist = false; + // if a concrete write index already exists, update the underlying mapping + if (concreteIndices.length > 0) { + await this.updateIndexMappings(esClient, concreteIndices); + + const concreteIndicesExist = concreteIndices.some( + (index) => index.alias === indexPatterns.alias + ); + concreteWriteIndicesExist = concreteIndices.some( + (index) => index.alias === indexPatterns.alias && index.isWriteIndex + ); + + // If there are some concrete indices but none of them are the write index, we'll throw an error + // because one of the existing indices should have been the write target. + if (concreteIndicesExist && !concreteWriteIndicesExist) { + throw new Error( + `Indices matching pattern ${indexPatterns.pattern} exist but none are set as the write index for alias ${indexPatterns.alias}` + ); + } + } + + // check if a concrete write index already exists + if (!concreteWriteIndicesExist) { + try { + await retryTransientEsErrors( + () => + esClient.indices.create({ + index: indexPatterns.name, + body: { + aliases: { + [indexPatterns.alias]: { + is_write_index: true, + }, + }, + }, + }), + { + logger: this.options.logger, + } + ); + } catch (error) { + this.options.logger.error(`Error creating concrete write index - ${error.message}`); + // If the index already exists and it's the write index for the alias, + // something else created it so suppress the error. If it's not the write + // index, that's bad, throw an error. + if (error?.meta?.body?.error?.type === 'resource_already_exists_exception') { + const existingIndices = await esClient.indices.get({ + index: indexPatterns.name, + }); + if ( + !existingIndices[indexPatterns.name]?.aliases?.[indexPatterns.alias]?.is_write_index + ) { + throw Error( + `Attempted to create index: ${indexPatterns.name} as the write index for alias: ${indexPatterns.alias}, but the index already exists and is not the write index for the alias` + ); + } + } else { + throw error; + } + } + } + } + + private async installWithTimeout( + installFn: () => Promise, + timeoutMs: number = INSTALLATION_TIMEOUT + ): Promise { + try { + let timeoutId: NodeJS.Timeout; + const install = async (): Promise => { + await installFn(); + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + + const throwTimeoutException = (): Promise => { + return new Promise((_, reject) => { + timeoutId = setTimeout(() => { + const msg = `Timeout: it took more than ${timeoutMs}ms`; + reject(new Error(msg)); + }, timeoutMs); + + firstValueFrom(this.options.pluginStop$).then(() => { + clearTimeout(timeoutId); + reject(new Error('Server is stopping; must stop all async operations')); + }); + }); + }; + + await Promise.race([install(), throwTimeoutException()]); + } catch (e) { + this.options.logger.error(e); + + const reason = e?.message || 'Unknown reason'; + throw new Error(`Failure during installation. ${reason}`); + } + } +} diff --git a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts new file mode 100644 index 0000000000000..f9ce460d04093 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { IRuleTypeAlerts } from '../types'; +import { createResourceInstallationHelper } from './create_resource_installation_helper'; + +const logger: ReturnType = + loggingSystemMock.createLogger(); + +const initFn = async (context: IRuleTypeAlerts, timeoutMs?: number) => { + logger.info(context.context); +}; + +const initFnWithDelay = async (context: IRuleTypeAlerts, timeoutMs?: number) => { + logger.info(context.context); + await new Promise((r) => setTimeout(r, 50)); +}; + +const initFnWithError = async (context: IRuleTypeAlerts, timeoutMs?: number) => { + throw new Error('fail'); +}; + +describe('createResourceInstallationHelper', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test(`should not call init function if readyToInitialize is false`, () => { + const helper = createResourceInstallationHelper(initFn); + + // Add two contexts that need to be initialized but don't call helper.setReadyToInitialize() + helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); + + expect(logger.info).not.toHaveBeenCalled(); + const initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(0); + }); + + test(`should call init function if readyToInitialize is set to true`, async () => { + const helper = createResourceInstallationHelper(initFn); + + // Add two contexts that need to be initialized and then call helper.setReadyToInitialize() + helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); + + helper.setReadyToInitialize(); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + expect(logger.info).toHaveBeenCalledTimes(2); + const initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(2); + + expect(await initializedContexts.get('test1')).toEqual(true); + expect(await initializedContexts.get('test2')).toEqual(true); + }); + + test(`should install resources for contexts added after readyToInitialize is called`, async () => { + const helper = createResourceInstallationHelper(initFnWithDelay); + + // Add two contexts that need to be initialized + helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); + + // Start processing the queued contexts + helper.setReadyToInitialize(); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + // Add another context to process + helper.add({ context: 'test3', fieldMap: { field: { type: 'keyword', required: false } } }); + + // 3 contexts with delay will take 150 + await new Promise((r) => setTimeout(r, 10)); + + expect(logger.info).toHaveBeenCalledTimes(3); + const initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(3); + + expect(await initializedContexts.get('test1')).toEqual(true); + expect(await initializedContexts.get('test2')).toEqual(true); + expect(await initializedContexts.get('test3')).toEqual(true); + }); + + test(`should install resources for contexts added after initial processing loop has run`, async () => { + const helper = createResourceInstallationHelper(initFn); + + // No contexts queued so this should finish quickly + helper.setReadyToInitialize(); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + expect(logger.info).not.toHaveBeenCalled(); + let initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(0); + + // Add a context to process + helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + expect(logger.info).toHaveBeenCalledTimes(1); + initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(1); + + expect(await initializedContexts.get('test1')).toEqual(true); + }); + + test(`should gracefully handle errors during initialization and set initialized flag to false`, async () => { + const helper = createResourceInstallationHelper(initFnWithError); + + helper.setReadyToInitialize(); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + // Add a context to process + helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + + // for the setImmediate + await new Promise((r) => setTimeout(r, 10)); + + const initializedContexts = helper.getInitializedContexts(); + expect([...initializedContexts.keys()].length).toEqual(1); + expect(await initializedContexts.get('test1')).toEqual(false); + }); +}); diff --git a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts new file mode 100644 index 0000000000000..0e3cbe0f87a9a --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRuleTypeAlerts } from '../types'; + +export interface ResourceInstallationHelper { + add: (context: IRuleTypeAlerts, timeoutMs?: number) => void; + setReadyToInitialize: (timeoutMs?: number) => void; + getInitializedContexts: () => Map>; +} + +/** + * Helper function that queues up resources to initialize until we are + * ready to begin initialization. Once we're ready, we start taking from + * the queue and kicking off initialization. + * + * If a resource is added after we begin initialization, we push it onto + * the queue and the running loop will handle it + * + * If a resource is added to the queue when the processing loop is not + * running, kick off the processing loop + */ +export function createResourceInstallationHelper( + initFn: (context: IRuleTypeAlerts, timeoutMs?: number) => Promise +): ResourceInstallationHelper { + let readyToInitialize = false; + let isInitializing: boolean = false; + const contextsToInitialize: IRuleTypeAlerts[] = []; + const initializedContexts: Map> = new Map(); + + const waitUntilContextResourcesInstalled = async ( + context: IRuleTypeAlerts, + timeoutMs?: number + ): Promise => { + try { + await initFn(context, timeoutMs); + return true; + } catch (err) { + return false; + } + }; + + const startInitialization = (timeoutMs?: number) => { + if (!readyToInitialize) { + return; + } + + setImmediate(async () => { + isInitializing = true; + while (contextsToInitialize.length > 0) { + const context = contextsToInitialize.pop()!; + initializedContexts.set( + context.context, + + // Return a promise than can be checked when needed + waitUntilContextResourcesInstalled(context, timeoutMs) + ); + } + isInitializing = false; + }); + }; + return { + add: (context: IRuleTypeAlerts, timeoutMs?: number) => { + contextsToInitialize.push(context); + if (!isInitializing) { + startInitialization(timeoutMs); + } + }, + setReadyToInitialize: (timeoutMs?: number) => { + readyToInitialize = true; + startInitialization(timeoutMs); + }, + getInitializedContexts: () => { + return initializedContexts; + }, + }; +} diff --git a/x-pack/plugins/alerting/server/alerts_service/default_lifecycle_policy.ts b/x-pack/plugins/alerting/server/alerts_service/default_lifecycle_policy.ts new file mode 100644 index 0000000000000..f5d11c6214ee6 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/default_lifecycle_policy.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Default alert index ILM policy + * - _meta.managed: notify users this is a managed policy and should be modified + * at their own risk + * - no delete phase as we want to keep these indices around indefinitely + * + * This should be used by all alerts-as-data indices + */ + +export const ILM_POLICY_NAME = 'alerts-default-ilm-policy'; +export const DEFAULT_ILM_POLICY = { + policy: { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.test.ts b/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.test.ts new file mode 100644 index 0000000000000..2501c57776d80 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { loggerMock } from '@kbn/logging-mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +const logger = loggerMock.create(); +const randomDelayMultiplier = 0.01; + +describe('retryTransientErrors', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it("doesn't retry if operation is successful", async () => { + const esCallMock = jest.fn().mockResolvedValue('success'); + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(1); + }); + + it('logs a warning message on retry', async () => { + const esCallMock = jest + .fn() + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockResolvedValue('success'); + + await retryTransientEsErrors(esCallMock, { logger }); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn.mock.calls[0][0]).toMatch( + `Retrying Elasticsearch operation after [2s] due to error: ConnectionError: foo ConnectionError: foo` + ); + }); + + it('retries with an exponential backoff', async () => { + let attempt = 0; + const esCallMock = jest.fn(async () => { + attempt++; + if (attempt < 4) { + throw new EsErrors.ConnectionError('foo'); + } else { + return 'success'; + } + }); + + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(4); + expect(logger.warn).toHaveBeenCalledTimes(3); + expect(logger.warn.mock.calls[0][0]).toMatch( + `Retrying Elasticsearch operation after [2s] due to error: ConnectionError: foo ConnectionError: foo` + ); + expect(logger.warn.mock.calls[1][0]).toMatch( + `Retrying Elasticsearch operation after [4s] due to error: ConnectionError: foo ConnectionError: foo` + ); + expect(logger.warn.mock.calls[2][0]).toMatch( + `Retrying Elasticsearch operation after [8s] due to error: ConnectionError: foo ConnectionError: foo` + ); + }); + + it('retries each supported error type', async () => { + const errors = [ + new EsErrors.NoLivingConnectionsError('no living connection', { + warnings: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + meta: {} as any, + }), + new EsErrors.ConnectionError('no connection'), + new EsErrors.TimeoutError('timeout'), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 503, meta: {} as any, warnings: [] }), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 408, meta: {} as any, warnings: [] }), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 410, meta: {} as any, warnings: [] }), + ]; + + for (const error of errors) { + const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success'); + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(2); + } + }); + + it('does not retry unsupported errors', async () => { + const error = new Error('foo!'); + const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success'); + await expect(retryTransientEsErrors(esCallMock, { logger })).rejects.toThrow(error); + expect(esCallMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.ts b/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.ts new file mode 100644 index 0000000000000..2df03e65690f1 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/retry_transient_es_errors.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/core/server'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +const MAX_ATTEMPTS = 3; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: Error) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { + logger, + attempt = 0, + }: { + logger: Logger; + attempt?: number; + } +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... + + logger.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + // delay with some randomness + await delay(retryDelaySec * 1000 * Math.random()); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/x-pack/plugins/alerting/server/alerts_service/types.ts b/x-pack/plugins/alerting/server/alerts_service/types.ts new file mode 100644 index 0000000000000..db47a9a8e0015 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { getComponentTemplateFromFieldMap } from '../../common/alert_schema'; +import { FieldMap } from '../../common/alert_schema/field_maps/types'; + +export const getComponentTemplateName = (context?: string) => + `alerts-${context ? context : 'common'}-component-template`; + +export interface IIndexPatternString { + template: string; + pattern: string; + alias: string; + name: string; +} + +export const getIndexTemplateAndPattern = ( + context: string, + namespace?: string +): IIndexPatternString => { + const pattern = `${context}-${namespace ? namespace : 'default'}`; + return { + template: `.alerts-${pattern}-template`, + pattern: `.alerts-${pattern}-*`, + alias: `.alerts-${pattern}`, + name: `.alerts-${pattern}-000001`, + }; +}; + +export const getComponentTemplate = ( + fieldMap: FieldMap, + context?: string +): ClusterPutComponentTemplateRequest => + getComponentTemplateFromFieldMap({ + name: getComponentTemplateName(context), + fieldMap, + // set field limit slightly higher than actual number of fields + fieldLimit: 100, // Math.round(Object.keys(fieldMap).length * 1.5), + }); diff --git a/x-pack/plugins/alerting/server/config.test.ts b/x-pack/plugins/alerting/server/config.test.ts index ec6f2f6565d67..26ea818719b7e 100644 --- a/x-pack/plugins/alerting/server/config.test.ts +++ b/x-pack/plugins/alerting/server/config.test.ts @@ -13,6 +13,7 @@ describe('config validation', () => { expect(configSchema.validate(config)).toMatchInlineSnapshot(` Object { "cancelAlertsOnRuleTimeout": true, + "enableFrameworkAlerts": false, "healthCheck": Object { "interval": "60m", }, diff --git a/x-pack/plugins/alerting/server/config.ts b/x-pack/plugins/alerting/server/config.ts index f6becbf192b05..f727cb98c0266 100644 --- a/x-pack/plugins/alerting/server/config.ts +++ b/x-pack/plugins/alerting/server/config.ts @@ -62,6 +62,7 @@ export const configSchema = schema.object({ maxEphemeralActionsPerAlert: schema.number({ defaultValue: DEFAULT_MAX_EPHEMERAL_ACTIONS_PER_ALERT, }), + enableFrameworkAlerts: schema.boolean({ defaultValue: false }), cancelAlertsOnRuleTimeout: schema.boolean({ defaultValue: true }), rules: rulesSchema, }); diff --git a/x-pack/plugins/alerting/server/plugin.test.ts b/x-pack/plugins/alerting/server/plugin.test.ts index 7b85bc898fee0..4e48c0db01ae6 100644 --- a/x-pack/plugins/alerting/server/plugin.test.ts +++ b/x-pack/plugins/alerting/server/plugin.test.ts @@ -26,6 +26,13 @@ import { PluginSetup as DataPluginSetup, } from '@kbn/data-plugin/server'; import { spacesMock } from '@kbn/spaces-plugin/server/mocks'; +import { AlertsService } from './alerts_service/alerts_service'; +import { alertsServiceMock } from './alerts_service/alerts_service.mock'; + +const mockAlertService = alertsServiceMock.create(); +jest.mock('./alerts_service/alerts_service', () => ({ + AlertsService: jest.fn().mockImplementation(() => mockAlertService), +})); import { SharePluginStart } from '@kbn/share-plugin/server'; import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; @@ -33,6 +40,7 @@ const generateAlertingConfig = (): AlertingConfig => ({ healthCheck: { interval: '5m', }, + enableFrameworkAlerts: false, invalidateApiKeysTask: { interval: '5m', removalDelay: '1h', @@ -116,6 +124,20 @@ describe('Alerting Plugin', () => { expect(usageCollectionSetup.registerCollector).toHaveBeenCalled(); }); + it('should initialize AlertsService if enableFrameworkAlerts config is true', async () => { + const context = coreMock.createPluginInitializerContext({ + ...generateAlertingConfig(), + enableFrameworkAlerts: true, + }); + plugin = new AlertingPlugin(context); + + // need await to test number of calls of setupMocks.status.set, because it is under async function which awaiting core.getStartServices() + await plugin.setup(setupMocks, mockPlugins); + + expect(AlertsService).toHaveBeenCalled(); + expect(mockAlertService.initialize).toHaveBeenCalled(); + }); + it(`exposes configured minimumScheduleInterval()`, async () => { const context = coreMock.createPluginInitializerContext( generateAlertingConfig() diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index b3b94584025c3..5ae4be21d9a63 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -6,7 +6,7 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { BehaviorSubject } from 'rxjs'; +import { BehaviorSubject, ReplaySubject, Subject } from 'rxjs'; import { pick } from 'lodash'; import { UsageCollectionSetup, UsageCounter } from '@kbn/usage-collection-plugin/server'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; @@ -86,6 +86,7 @@ import { getSecurityHealth, SecurityHealth } from './lib/get_security_health'; import { registerNodeCollector, registerClusterCollector, InMemoryMetrics } from './monitoring'; import { getRuleTaskTimeout } from './lib/get_rule_task_timeout'; import { getActionsConfigMap } from './lib/get_actions_config_map'; +import { AlertsService } from './alerts_service/alerts_service'; import { rulesSettingsFeature } from './rules_settings_feature'; export const EVENT_LOG_PROVIDER = 'alerting'; @@ -186,6 +187,8 @@ export class AlertingPlugin { private kibanaBaseUrl: string | undefined; private usageCounter: UsageCounter | undefined; private inMemoryMetrics: InMemoryMetrics; + private alertsService?: AlertsService; + private pluginStop$: Subject; constructor(initializerContext: PluginInitializerContext) { this.config = initializerContext.config.get(); @@ -197,6 +200,7 @@ export class AlertingPlugin { this.telemetryLogger = initializerContext.logger.get('usage'); this.kibanaVersion = initializerContext.env.packageInfo.version; this.inMemoryMetrics = new InMemoryMetrics(initializerContext.logger.get('in_memory_metrics')); + this.pluginStop$ = new ReplaySubject(1); } public setup( @@ -235,12 +239,24 @@ export class AlertingPlugin { this.eventLogService = plugins.eventLog; plugins.eventLog.registerProviderActions(EVENT_LOG_PROVIDER, Object.values(EVENT_LOG_ACTIONS)); + if (this.config.enableFrameworkAlerts) { + this.alertsService = new AlertsService({ + logger: this.logger, + pluginStop$: this.pluginStop$, + elasticsearchClientPromise: core + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + }); + this.alertsService!.initialize(); + } + const ruleTypeRegistry = new RuleTypeRegistry({ logger: this.logger, taskManager: plugins.taskManager, taskRunnerFactory: this.taskRunnerFactory, licenseState: this.licenseState, licensing: plugins.licensing, + alertsService: this.alertsService, minimumScheduleInterval: this.config.rules.minimumScheduleInterval, inMemoryMetrics: this.inMemoryMetrics, }); @@ -526,5 +542,7 @@ export class AlertingPlugin { if (this.licenseState) { this.licenseState.clean(); } + this.pluginStop$.next(); + this.pluginStop$.complete(); } } diff --git a/x-pack/plugins/alerting/server/rule_type_registry.test.ts b/x-pack/plugins/alerting/server/rule_type_registry.test.ts index 8391d94576d13..bb15ef287e408 100644 --- a/x-pack/plugins/alerting/server/rule_type_registry.test.ts +++ b/x-pack/plugins/alerting/server/rule_type_registry.test.ts @@ -14,6 +14,7 @@ import { licenseStateMock } from './lib/license_state.mock'; import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { loggingSystemMock } from '@kbn/core/server/mocks'; import { inMemoryMetricsMock } from './monitoring/in_memory_metrics.mock'; +import { alertsServiceMock } from './alerts_service/alerts_service.mock'; const logger = loggingSystemMock.create().get(); let mockedLicenseState: jest.Mocked; @@ -21,6 +22,7 @@ let ruleTypeRegistryParams: ConstructorOptions; const taskManager = taskManagerMock.createSetup(); const inMemoryMetrics = inMemoryMetricsMock.create(); +const alertsService = alertsServiceMock.create(); beforeEach(() => { jest.resetAllMocks(); @@ -451,6 +453,55 @@ describe('Create Lifecycle', () => { }) ).toThrowErrorMatchingInlineSnapshot(`"Rule type \\"test\\" is already registered."`); }); + + test('should initialize alerts as data resources if AlertsService is defined and alert definition is registered', () => { + const registry = new RuleTypeRegistry({ ...ruleTypeRegistryParams, alertsService }); + registry.register({ + id: 'test', + name: 'Test', + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + executor: jest.fn(), + producer: 'alerts', + alerts: { + context: 'test', + fieldMap: { field: { type: 'keyword', required: false } }, + }, + }); + + expect(alertsService.register).toHaveBeenCalledWith({ + context: 'test', + fieldMap: { field: { type: 'keyword', required: false } }, + }); + }); + + test('should not initialize alerts as data resources if no alert definition is registered', () => { + const registry = new RuleTypeRegistry({ ...ruleTypeRegistryParams, alertsService }); + registry.register({ + id: 'test', + name: 'Test', + actionGroups: [ + { + id: 'default', + name: 'Default', + }, + ], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + executor: jest.fn(), + producer: 'alerts', + }); + + expect(alertsService.register).not.toHaveBeenCalled(); + }); }); describe('get()', () => { diff --git a/x-pack/plugins/alerting/server/rule_type_registry.ts b/x-pack/plugins/alerting/server/rule_type_registry.ts index b908f7cb67b87..b4be0957f11af 100644 --- a/x-pack/plugins/alerting/server/rule_type_registry.ts +++ b/x-pack/plugins/alerting/server/rule_type_registry.ts @@ -33,6 +33,7 @@ import { ILicenseState } from './lib/license_state'; import { getRuleTypeFeatureUsageName } from './lib/get_rule_type_feature_usage_name'; import { InMemoryMetrics } from './monitoring'; import { AlertingRulesConfig } from '.'; +import { AlertsService } from './alerts_service/alerts_service'; export interface ConstructorOptions { logger: Logger; @@ -42,6 +43,7 @@ export interface ConstructorOptions { licensing: LicensingPluginSetup; minimumScheduleInterval: AlertingRulesConfig['minimumScheduleInterval']; inMemoryMetrics: InMemoryMetrics; + alertsService?: AlertsService; } export interface RegistryRuleType @@ -139,6 +141,7 @@ export class RuleTypeRegistry { private readonly minimumScheduleInterval: AlertingRulesConfig['minimumScheduleInterval']; private readonly licensing: LicensingPluginSetup; private readonly inMemoryMetrics: InMemoryMetrics; + private readonly alertsService?: AlertsService; constructor({ logger, @@ -148,6 +151,7 @@ export class RuleTypeRegistry { licensing, minimumScheduleInterval, inMemoryMetrics, + alertsService, }: ConstructorOptions) { this.logger = logger; this.taskManager = taskManager; @@ -156,6 +160,7 @@ export class RuleTypeRegistry { this.licensing = licensing; this.minimumScheduleInterval = minimumScheduleInterval; this.inMemoryMetrics = inMemoryMetrics; + this.alertsService = alertsService; } public has(id: string) { @@ -277,6 +282,11 @@ export class RuleTypeRegistry { >(normalizedRuleType, context, this.inMemoryMetrics), }, }); + + if (this.alertsService && ruleType.alerts) { + this.alertsService.register(ruleType.alerts); + } + // No need to notify usage on basic alert types if (ruleType.minimumLicenseRequired !== 'basic') { this.licensing.featureUsage.register( diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index f4b7cc57b27e4..4092b9209432a 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -50,6 +50,7 @@ import { RuleLastRun, } from '../common'; import { PublicAlertFactory } from './alert/create_alert_factory'; +import { FieldMap } from '../common/alert_schema/field_maps/types'; export type WithoutQueryAndParams = Pick>; export type SpaceIdToNamespaceFunction = (spaceId?: string) => string | undefined; export type { RuleTypeParams }; @@ -158,6 +159,11 @@ export interface SummarizedAlerts { }; } export type GetSummarizedAlertsFn = (opts: GetSummarizedAlertsFnOpts) => Promise; +export interface IRuleTypeAlerts { + context: string; + namespace?: string; + fieldMap: FieldMap; +} export interface RuleType< Params extends RuleTypeParams = never, @@ -204,6 +210,7 @@ export interface RuleType< cancelAlertsOnRuleTimeout?: boolean; doesSetRecoveryContext?: boolean; getSummarizedAlerts?: GetSummarizedAlertsFn; + alerts?: IRuleTypeAlerts; /** * Determines whether framework should * automatically make recovery determination. Defaults to true. diff --git a/x-pack/plugins/alerting/tsconfig.json b/x-pack/plugins/alerting/tsconfig.json index a1fb0d3892d2a..73c407bd325a8 100644 --- a/x-pack/plugins/alerting/tsconfig.json +++ b/x-pack/plugins/alerting/tsconfig.json @@ -8,7 +8,7 @@ // have to declare *.json explicitly due to https://github.com/microsoft/TypeScript/issues/25636 "server/**/*.json", "public/**/*", - "common/*" + "common/**/*" ], "kbn_references": [ "@kbn/core", @@ -39,6 +39,7 @@ "@kbn/apm-utils", "@kbn/data-views-plugin", "@kbn/share-plugin", + "@kbn/safer-lodash-set", ], "exclude": [ "target/**/*", diff --git a/x-pack/test/alerting_api_integration/common/config.ts b/x-pack/test/alerting_api_integration/common/config.ts index 2f70ad5902483..6ee90026d9821 100644 --- a/x-pack/test/alerting_api_integration/common/config.ts +++ b/x-pack/test/alerting_api_integration/common/config.ts @@ -185,6 +185,7 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) `--xpack.alerting.rules.run.actions.connectorTypeOverrides=${JSON.stringify([ { id: 'test.capped', max: '1' }, ])}`, + `--xpack.alerting.enableFrameworkAlerts=true`, `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`, `--xpack.actions.rejectUnauthorized=${rejectUnauthorized}`, `--xpack.actions.microsoftGraphApiUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}/api/_actions-FTS-external-service-simulators/exchange/users/test@/sendMail`, diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts index 0c8390ca938dc..3c2880d69f776 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts @@ -92,6 +92,27 @@ function getAlwaysFiringAlertType() { context: [{ name: 'instanceContextValue', description: 'the instance context value' }], }, executor: curry(alwaysFiringExecutor)(), + alerts: { + context: 'test.always-firing', + fieldMap: { + instance_state_value: { + required: false, + type: 'boolean', + }, + instance_params_value: { + required: false, + type: 'boolean', + }, + instance_context_value: { + required: false, + type: 'boolean', + }, + group_in_series_index: { + required: false, + type: 'long', + }, + }, + }, }; return result; } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts_as_data.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts_as_data.ts new file mode 100644 index 0000000000000..13cb9bcd337f9 --- /dev/null +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts_as_data.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertFieldMap } from '@kbn/alerting-plugin/common/alert_schema'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common/alert_schema/field_maps/mapping_from_field_map'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function createAlertsAsDataTest({ getService }: FtrProviderContext) { + const es = getService('es'); + const commonFrameworkMappings = mappingFromFieldMap(alertFieldMap, 'strict'); + + describe('alerts as data', () => { + it('should install common alerts as data resources on startup', async () => { + const ilmPolicyName = 'alerts-default-ilm-policy'; + const componentTemplateName = 'alerts-common-component-template'; + + const commonIlmPolicy = await es.ilm.getLifecycle({ + name: ilmPolicyName, + }); + + expect(commonIlmPolicy[ilmPolicyName].policy).to.eql({ + _meta: { + managed: true, + }, + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }); + + const { component_templates: componentTemplates } = await es.cluster.getComponentTemplate({ + name: componentTemplateName, + }); + + expect(componentTemplates.length).to.eql(1); + const commonComponentTemplate = componentTemplates[0]; + + expect(commonComponentTemplate.name).to.eql(componentTemplateName); + expect(commonComponentTemplate.component_template.template.mappings).to.eql( + commonFrameworkMappings + ); + expect(commonComponentTemplate.component_template.template.settings).to.eql({ + index: { + number_of_shards: 1, + mapping: { + total_fields: { + limit: 100, + }, + }, + }, + }); + }); + + it('should install context specific alerts as data resources on startup', async () => { + const componentTemplateName = 'alerts-test.always-firing-component-template'; + const indexTemplateName = '.alerts-test.always-firing-default-template'; + const indexName = '.alerts-test.always-firing-default-000001'; + const contextSpecificMappings = { + instance_params_value: { + type: 'boolean', + }, + instance_state_value: { + type: 'boolean', + }, + instance_context_value: { + type: 'boolean', + }, + group_in_series_index: { + type: 'long', + }, + }; + + const { component_templates: componentTemplates } = await es.cluster.getComponentTemplate({ + name: componentTemplateName, + }); + expect(componentTemplates.length).to.eql(1); + const contextComponentTemplate = componentTemplates[0]; + expect(contextComponentTemplate.name).to.eql(componentTemplateName); + expect(contextComponentTemplate.component_template.template.mappings).to.eql({ + dynamic: 'strict', + properties: contextSpecificMappings, + }); + expect(contextComponentTemplate.component_template.template.settings).to.eql({ + index: { + number_of_shards: 1, + mapping: { + total_fields: { + limit: 100, + }, + }, + }, + }); + + const { index_templates: indexTemplates } = await es.indices.getIndexTemplate({ + name: indexTemplateName, + }); + expect(indexTemplates.length).to.eql(1); + const contextIndexTemplate = indexTemplates[0]; + expect(contextIndexTemplate.name).to.eql(indexTemplateName); + expect(contextIndexTemplate.index_template.index_patterns).to.eql([ + '.alerts-test.always-firing-default-*', + ]); + expect(contextIndexTemplate.index_template.composed_of).to.eql([ + 'alerts-common-component-template', + 'alerts-test.always-firing-component-template', + ]); + expect(contextIndexTemplate.index_template.template!.mappings).to.eql({ + dynamic: false, + }); + expect(contextIndexTemplate.index_template.template!.settings).to.eql({ + index: { + lifecycle: { + name: 'alerts-default-ilm-policy', + rollover_alias: '.alerts-test.always-firing-default', + }, + mapping: { + total_fields: { + limit: '2500', + }, + }, + hidden: 'true', + auto_expand_replicas: '0-1', + }, + }); + + const contextIndex = await es.indices.get({ + index: indexName, + }); + + expect(contextIndex[indexName].aliases).to.eql({ + '.alerts-test.always-firing-default': { + is_write_index: true, + }, + }); + + expect(contextIndex[indexName].mappings).to.eql({ + dynamic: 'false', + properties: { + ...contextSpecificMappings, + ...commonFrameworkMappings.properties, + }, + }); + + expect(contextIndex[indexName].settings?.index?.lifecycle).to.eql({ + name: 'alerts-default-ilm-policy', + rollover_alias: '.alerts-test.always-firing-default', + }); + + expect(contextIndex[indexName].settings?.index?.mapping).to.eql({ + total_fields: { + limit: '2500', + }, + }); + + expect(contextIndex[indexName].settings?.index?.hidden).to.eql('true'); + expect(contextIndex[indexName].settings?.index?.number_of_shards).to.eql(1); + expect(contextIndex[indexName].settings?.index?.auto_expand_replicas).to.eql('0-1'); + expect(contextIndex[indexName].settings?.index?.provided_name).to.eql( + '.alerts-test.always-firing-default-000001' + ); + }); + }); +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts index 9f3faf1d3a5fa..d7da90cf56df4 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/index.ts @@ -53,6 +53,7 @@ export default function alertingTests({ loadTestFile, getService }: FtrProviderC loadTestFile(require.resolve('./run_soon')); loadTestFile(require.resolve('./flapping_history')); loadTestFile(require.resolve('./check_registered_rule_types')); + loadTestFile(require.resolve('./alerts_as_data')); // Do not place test files here, due to https://github.com/elastic/kibana/issues/123059 // note that this test will destroy existing spaces