diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 7b5c3b5337c02..7bd76838c7559 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -27,7 +27,8 @@ "spaces", "usageCollection", "lists", - "home" + "home", + "telemetry" ], "server": true, "ui": true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 68c6a51b4e6f6..c82c1fe969ee3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -130,6 +130,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -255,6 +256,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -334,6 +336,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -394,6 +397,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -452,6 +456,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -535,6 +540,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [exceptionItem], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -615,6 +621,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -664,6 +671,7 @@ describe('searchAfterAndBulkCreate', () => { ruleParams: sampleParams, services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -719,6 +727,7 @@ describe('searchAfterAndBulkCreate', () => { ruleParams: sampleParams, services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -790,6 +799,7 @@ describe('searchAfterAndBulkCreate', () => { ruleParams: sampleParams, services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, @@ -898,6 +908,7 @@ describe('searchAfterAndBulkCreate', () => { exceptionsList: [], services: mockService, logger: mockLogger, + eventsTelemetry: undefined, id: sampleRuleGuid, inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 2df180582a0ac..8fe55d97b569c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -7,6 +7,7 @@ import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filter_events_with_list'; +import { sendAlertTelemetryEvents } from './send_telemetry_events'; import { createSearchAfterReturnType, createSearchAfterReturnTypeFromResponse, @@ -25,6 +26,7 @@ export const searchAfterAndBulkCreate = async ({ services, listClient, logger, + eventsTelemetry, id, inputIndexPattern, signalsIndex, @@ -188,6 +190,14 @@ export const searchAfterAndBulkCreate = async ({ logger.debug( buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`) ); + + sendAlertTelemetryEvents( + logger, + eventsTelemetry, + filteredEvents, + ruleParams, + buildRuleMessage + ); } // we are guaranteed to have searchResult hits at this point diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts new file mode 100644 index 0000000000000..2a531998ff8a6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { selectEvents } from './send_telemetry_events'; + +describe('sendAlertTelemetry', () => { + it('selectEvents', () => { + const filteredEvents = { + took: 0, + timed_out: false, + _shards: { + total: 1, + successful: 1, + failed: 0, + skipped: 0, + }, + hits: { + total: 2, + max_score: 0, + hits: [ + { + _index: 'x', + _type: 'x', + _id: 'x', + _score: 0, + _source: { + '@timestamp': 'x', + key1: 'hello', + data_stream: { + dataset: 'endpoint.events', + }, + }, + }, + { + _index: 'x', + _type: 'x', + _id: 'x', + _score: 0, + _source: { + '@timestamp': 'x', + key2: 'hello', + data_stream: { + dataset: 'endpoint.alerts', + other: 'x', + }, + }, + }, + { + _index: 'x', + _type: 'x', + _id: 'x', + _score: 0, + _source: { + '@timestamp': 'x', + key3: 'hello', + data_stream: {}, + }, + }, + ], + }, + }; + + const sources = selectEvents(filteredEvents); + expect(sources).toStrictEqual([ + { + '@timestamp': 'x', + key2: 'hello', + data_stream: { + dataset: 'endpoint.alerts', + other: 'x', + }, + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts new file mode 100644 index 0000000000000..5963d31bda8a6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TelemetryEventsSender, TelemetryEvent } from '../../telemetry/sender'; +import { RuleTypeParams } from '../types'; +import { BuildRuleMessage } from './rule_messages'; +import { SignalSearchResponse, SignalSource } from './types'; +import { Logger } from '../../../../../../../src/core/server'; + +export interface SearchResultWithSource { + _source: SignalSource; +} + +export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEvent[] { + const sources = filteredEvents.hits.hits.map(function ( + obj: SearchResultWithSource + ): TelemetryEvent { + return obj._source; + }); + + // Filter out non-endpoint alerts + return sources.filter((obj: TelemetryEvent) => obj.data_stream?.dataset === 'endpoint.alerts'); +} + +export function sendAlertTelemetryEvents( + logger: Logger, + eventsTelemetry: TelemetryEventsSender | undefined, + filteredEvents: SignalSearchResponse, + ruleParams: RuleTypeParams, + buildRuleMessage: BuildRuleMessage +) { + if (eventsTelemetry === undefined) { + return; + } + + const sources = selectEvents(filteredEvents); + + try { + eventsTelemetry.queueTelemetryEvents(sources); + } catch (exc) { + logger.error(buildRuleMessage(`[-] queing telemetry events failed ${exc}`)); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 382acf2f38245..415abc9d995fb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -138,6 +138,7 @@ describe('rules_notification_alert_type', () => { alert = signalRulesAlertType({ logger, + eventsTelemetry: undefined, version, ml: mlMock, lists: listMock.createSetup(), @@ -344,6 +345,7 @@ describe('rules_notification_alert_type', () => { payload = getPayload(ruleAlert, alertServices) as jest.Mocked; alert = signalRulesAlertType({ logger, + eventsTelemetry: undefined, version, ml: undefined, lists: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a3b37270e50b1..bf45e60d258fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -56,6 +56,7 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { TelemetryEventsSender } from '../../telemetry/sender'; import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; import { bulkInsertSignals } from './single_bulk_create'; import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; @@ -63,11 +64,13 @@ import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ logger, + eventsTelemetry, version, ml, lists, }: { logger: Logger; + eventsTelemetry: TelemetryEventsSender | undefined; version: string; ml: SetupPlugins['ml']; lists: SetupPlugins['lists'] | undefined; @@ -369,6 +372,7 @@ export const signalRulesAlertType = ({ previousStartedAt, listClient, logger, + eventsTelemetry, alertId, outputIndex, params, @@ -409,6 +413,7 @@ export const signalRulesAlertType = ({ ruleParams: params, services, logger, + eventsTelemetry, id: alertId, inputIndexPattern: inputIndex, signalsIndex: outputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index a6d4a2ba58ddd..560e7ad7fe2cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -28,6 +28,7 @@ export const createThreatSignal = async ({ previousStartedAt, listClient, logger, + eventsTelemetry, alertId, outputIndex, params, @@ -77,6 +78,7 @@ export const createThreatSignal = async ({ ruleParams: params, services, logger, + eventsTelemetry, id: alertId, inputIndexPattern: inputIndex, signalsIndex: outputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index f416ae6703b66..f44c7a9684457 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -24,6 +24,7 @@ export const createThreatSignals = async ({ previousStartedAt, listClient, logger, + eventsTelemetry, alertId, outputIndex, params, @@ -79,6 +80,7 @@ export const createThreatSignals = async ({ previousStartedAt, listClient, logger, + eventsTelemetry, alertId, outputIndex, params, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index d63f2d2b3b6aa..7cd6e5196ea68 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -21,6 +21,7 @@ import { AlertServices } from '../../../../../../alerts/server'; import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas'; import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/core/server'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; +import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { SearchAfterAndBulkCreateReturnType } from '../types'; @@ -38,6 +39,7 @@ export interface CreateThreatSignalsOptions { previousStartedAt: Date | null; listClient: ListClient; logger: Logger; + eventsTelemetry: TelemetryEventsSender | undefined; alertId: string; outputIndex: string; params: RuleTypeParams; @@ -73,6 +75,7 @@ export interface CreateThreatSignalOptions { previousStartedAt: Date | null; listClient: ListClient; logger: Logger; + eventsTelemetry: TelemetryEventsSender | undefined; alertId: string; outputIndex: string; params: RuleTypeParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 2f6ed0c1e3a8e..d9ca3dce54af3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -21,6 +21,7 @@ import { ListClient } from '../../../../../lists/server'; import { Logger } from '../../../../../../../src/core/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { BuildRuleMessage } from './rule_messages'; +import { TelemetryEventsSender } from '../../telemetry/sender'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -211,6 +212,7 @@ export interface SearchAfterAndBulkCreateParams { listClient: ListClient; exceptionsList: ExceptionListItemSchema[]; logger: Logger; + eventsTelemetry: TelemetryEventsSender | undefined; id: string; inputIndexPattern: string[]; signalsIndex: string; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts new file mode 100644 index 0000000000000..1ebdcb6b9d3f4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable dot-notation */ +import { TelemetryEventsSender, copyAllowlistedFields, getV3UrlFromV2 } from './sender'; +import { loggingSystemMock } from 'src/core/server/mocks'; +import { URL } from 'url'; + +describe('TelemetryEventsSender', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + describe('processEvents', () => { + it('returns empty array when empty array is passed', () => { + const sender = new TelemetryEventsSender(logger); + const result = sender.processEvents([]); + expect(result).toStrictEqual([]); + }); + + it('applies the allowlist', () => { + const sender = new TelemetryEventsSender(logger); + const input = [ + { + event: { + kind: 'alert', + }, + agent: { + name: 'test', + }, + file: { + size: 3, + path: 'X', + test: 'me', + another: 'nope', + Ext: { + code_signature: { + key1: 'X', + key2: 'Y', + }, + malware_classification: { + key1: 'X', + }, + something_else: 'nope', + }, + }, + host: { + os: { + name: 'windows', + }, + something_else: 'nope', + }, + }, + ]; + + const result = sender.processEvents(input); + expect(result).toStrictEqual([ + { + event: { + kind: 'alert', + }, + agent: { + name: 'test', + }, + file: { + size: 3, + path: 'X', + Ext: { + code_signature: { + key1: 'X', + key2: 'Y', + }, + malware_classification: { + key1: 'X', + }, + }, + }, + host: { + os: { + name: 'windows', + }, + }, + }, + ]); + }); + }); + + describe('queueTelemetryEvents', () => { + it('queues two events', () => { + const sender = new TelemetryEventsSender(logger); + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + }); + + it('queues more than maxQueueSize events', () => { + const sender = new TelemetryEventsSender(logger); + sender['maxQueueSize'] = 5; + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + sender.queueTelemetryEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]); + expect(sender['queue'].length).toBe(5); + }); + + it('empties the queue when sending', async () => { + const sender = new TelemetryEventsSender(logger); + sender['sendEvents'] = jest.fn(); + sender['telemetryStart'] = { + getIsOptedIn: jest.fn(async () => true), + }; + sender['telemetrySetup'] = { + getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), + }; + sender['fetchClusterInfo'] = jest.fn(async () => { + return { + cluster_name: 'test', + cluster_uuid: 'test-uuid', + }; + }); + + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + await sender['sendIfDue'](); + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(1); + sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]); + sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]); + expect(sender['queue'].length).toBe(4); + await sender['sendIfDue'](); + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(2); + }); + + it("shouldn't send when telemetry is disabled", async () => { + const sender = new TelemetryEventsSender(logger); + sender['sendEvents'] = jest.fn(); + const telemetryStart = { + getIsOptedIn: jest.fn(async () => false), + }; + sender['telemetryStart'] = telemetryStart; + + sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]); + expect(sender['queue'].length).toBe(2); + await sender['sendIfDue'](); + + expect(sender['queue'].length).toBe(0); + expect(sender['sendEvents']).toBeCalledTimes(0); + }); + }); +}); + +describe('allowlistEventFields', () => { + const allowlist = { + a: true, + b: true, + c: { + d: true, + }, + }; + + it('filters top level', () => { + const event = { + a: 'a', + a1: 'a1', + b: 'b', + b1: 'b1', + }; + expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({ + a: 'a', + b: 'b', + }); + }); + + it('filters nested', () => { + const event = { + a: { + a1: 'a1', + }, + a1: 'a1', + b: { + b1: 'b1', + }, + b1: 'b1', + c: { + d: 'd', + e: 'e', + f: 'f', + }, + }; + expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({ + a: { + a1: 'a1', + }, + b: { + b1: 'b1', + }, + c: { + d: 'd', + }, + }); + }); + + it("doesn't create empty objects", () => { + const event = { + a: 'a', + b: 'b', + c: { + e: 'e', + }, + }; + expect(copyAllowlistedFields(allowlist, event)).toStrictEqual({ + a: 'a', + b: 'b', + }); + }); +}); + +describe('getV3UrlFromV2', () => { + it('should return prod url', () => { + expect(getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint')).toBe( + 'https://telemetry.elastic.co/v3/send/alerts-endpoint' + ); + }); + + it('should return staging url', () => { + expect( + getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') + ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint'); + }); + + it('should support ports and auth', () => { + expect( + getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') + ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint'); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts new file mode 100644 index 0000000000000..acee75abddcd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -0,0 +1,368 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { cloneDeep } from 'lodash'; +import axios from 'axios'; +import { LegacyAPICaller } from 'kibana/server'; +import { URL } from 'url'; +import { Logger, CoreStart } from '../../../../../../src/core/server'; +import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson'; +import { + TelemetryPluginStart, + TelemetryPluginSetup, +} from '../../../../../../src/plugins/telemetry/server'; + +export type SearchTypes = + | string + | string[] + | number + | number[] + | boolean + | boolean[] + | object + | object[] + | undefined; + +export interface TelemetryEvent { + [key: string]: SearchTypes; + '@timestamp'?: string; + data_stream?: { + [key: string]: SearchTypes; + dataset?: string; + }; + cluster_name?: string; + cluster_uuid?: string; + file?: { + [key: string]: SearchTypes; + Ext?: { + [key: string]: SearchTypes; + }; + }; + license?: ESLicense; +} + +export class TelemetryEventsSender { + private readonly initialCheckDelayMs = 10 * 1000; + private readonly checkIntervalMs = 60 * 1000; + private readonly logger: Logger; + private core?: CoreStart; + private maxQueueSize = 100; + private telemetryStart?: TelemetryPluginStart; + private telemetrySetup?: TelemetryPluginSetup; + private intervalId?: NodeJS.Timeout; + private isSending = false; + private queue: TelemetryEvent[] = []; + private isOptedIn?: boolean = true; // Assume true until the first check + + constructor(logger: Logger) { + this.logger = logger.get('telemetry_events'); + } + + public setup(telemetrySetup?: TelemetryPluginSetup) { + this.telemetrySetup = telemetrySetup; + } + + public start(core?: CoreStart, telemetryStart?: TelemetryPluginStart) { + this.telemetryStart = telemetryStart; + this.core = core; + + this.logger.debug(`Starting task`); + setTimeout(() => { + this.sendIfDue(); + this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs); + }, this.initialCheckDelayMs); + } + + public stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + } + } + + public queueTelemetryEvents(events: TelemetryEvent[]) { + const qlength = this.queue.length; + + if (events.length === 0) { + return; + } + + this.logger.debug(`Queue events`); + + if (qlength >= this.maxQueueSize) { + // we're full already + return; + } + + if (events.length > this.maxQueueSize - qlength) { + this.queue.push(...this.processEvents(events.slice(0, this.maxQueueSize - qlength))); + } else { + this.queue.push(...this.processEvents(events)); + } + } + + public processEvents(events: TelemetryEvent[]): TelemetryEvent[] { + return events.map(function (obj: TelemetryEvent): TelemetryEvent { + return copyAllowlistedFields(allowlistEventFields, obj); + }); + } + + private async sendIfDue() { + if (this.isSending) { + return; + } + + if (this.queue.length === 0) { + return; + } + + try { + this.isSending = true; + + // Checking opt-in status is relatively expensive (calls a saved-object), so + // we only check it when we have things to send. + this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); + if (!this.isOptedIn) { + this.logger.debug(`Telemetry is not opted-in.`); + this.queue = []; + this.isSending = false; + return; + } + + const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ + this.fetchTelemetryUrl(), + this.fetchClusterInfo(), + this.fetchLicenseInfo(), + ]); + + this.logger.debug(`Telemetry URL: ${telemetryUrl}`); + this.logger.debug( + `cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}` + ); + + const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ + ...event, + ...(licenseInfo ? { license: this.copyLicenseFields(licenseInfo) } : {}), + cluster_uuid: clusterInfo.cluster_uuid, + cluster_name: clusterInfo.cluster_name, + })); + this.queue = []; + + await this.sendEvents(toSend, telemetryUrl, clusterInfo.cluster_uuid, licenseInfo?.uid); + } catch (err) { + this.logger.warn(`Error sending telemetry events data: ${err}`); + this.queue = []; + } + this.isSending = false; + } + + private async fetchClusterInfo(): Promise { + if (!this.core) { + throw Error("Couldn't fetch cluster info because core is not available"); + } + const callCluster = this.core.elasticsearch.legacy.client.callAsInternalUser; + return getClusterInfo(callCluster); + } + + private async fetchTelemetryUrl(): Promise { + const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); + if (!telemetryUrl) { + throw Error("Couldn't get telemetry URL"); + } + return getV3UrlFromV2(telemetryUrl.toString(), 'alerts-endpoint'); + } + + private async fetchLicenseInfo(): Promise { + if (!this.core) { + return undefined; + } + try { + const callCluster = this.core.elasticsearch.legacy.client.callAsInternalUser; + const ret = await getLicense(callCluster, true); + return ret.license; + } catch (err) { + this.logger.warn(`Error retrieving license: ${err}`); + return undefined; + } + } + + private copyLicenseFields(lic: ESLicense) { + return { + uid: lic.uid, + status: lic.status, + type: lic.type, + ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), + ...(lic.issuer ? { issuer: lic.issuer } : {}), + }; + } + + private async sendEvents( + events: unknown[], + telemetryUrl: string, + clusterUuid: string, + licenseId: string | undefined + ) { + // this.logger.debug(`Sending events: ${JSON.stringify(events, null, 2)}`); + const ndjson = transformDataToNdjson(events); + // this.logger.debug(`NDJSON: ${ndjson}`); + + try { + const resp = await axios.post(telemetryUrl, ndjson, { + headers: { + 'Content-Type': 'application/x-ndjson', + 'X-Elastic-Cluster-ID': clusterUuid, + ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}), + 'X-Elastic-Telemetry': '1', // TODO: no longer needed? + }, + }); + this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`); + } catch (err) { + this.logger.warn( + `Error sending events: ${err.response.status} ${JSON.stringify(err.response.data)}` + ); + } + } +} + +// For the Allowlist definition. +interface AllowlistFields { + [key: string]: boolean | AllowlistFields; +} + +// Allow list for the data we include in the events. True means that it is deep-cloned +// blindly. Object contents means that we only copy the fields that appear explicitly in +// the sub-object. +const allowlistEventFields: AllowlistFields = { + '@timestamp': true, + agent: true, + Endpoint: true, + ecs: true, + elastic: true, + event: true, + file: { + name: true, + path: true, + size: true, + created: true, + accessed: true, + mtime: true, + directory: true, + hash: true, + Ext: { + code_signature: true, + malware_classification: true, + }, + }, + host: { + os: true, + }, + process: { + name: true, + executable: true, + command_line: true, + hash: true, + Ext: { + code_signature: true, + }, + parent: { + name: true, + executable: true, + command_line: true, + hash: true, + Ext: { + code_signature: true, + }, + }, + }, +}; + +export function copyAllowlistedFields( + allowlist: AllowlistFields, + event: TelemetryEvent +): TelemetryEvent { + return Object.entries(allowlist).reduce((newEvent, [allowKey, allowValue]) => { + const eventValue = event[allowKey]; + if (eventValue) { + if (allowValue === true) { + return { ...newEvent, [allowKey]: eventValue }; + } else if (typeof allowValue === 'object' && typeof eventValue === 'object') { + const values = copyAllowlistedFields(allowValue, eventValue as TelemetryEvent); + return { + ...newEvent, + ...(Object.keys(values).length > 0 ? { [allowKey]: values } : {}), + }; + } + } + return newEvent; + }, {}); +} + +// Forms URLs like: +// https://telemetry.elastic.co/v3/send/my-channel-name or +// https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name +export function getV3UrlFromV2(v2url: string, channel: string): string { + const url = new URL(v2url); + if (!url.hostname.includes('staging')) { + url.pathname = `/v3/send/${channel}`; + } else { + url.pathname = `/v3-dev/send/${channel}`; + } + return url.toString(); +} + +// For getting cluster info. Copied from telemetry_collection/get_cluster_info.ts +export interface ESClusterInfo { + cluster_uuid: string; + cluster_name: string; + version?: { + number: string; + build_flavor: string; + build_type: string; + build_hash: string; + build_date: string; + build_snapshot?: boolean; + lucene_version: string; + minimum_wire_compatibility_version: string; + minimum_index_compatibility_version: string; + }; +} + +/** + * Get the cluster info from the connected cluster. + * + * This is the equivalent to GET / + * + * @param {function} callCluster The callWithInternalUser handler (exposed for testing) + */ +function getClusterInfo(callCluster: LegacyAPICaller) { + return callCluster('info'); +} + +// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html +export interface ESLicense { + status: string; + uid: string; + type: string; + issue_date?: string; + issue_date_in_millis?: number; + expiry_date?: string; + expirty_date_in_millis?: number; + max_nodes?: number; + issued_to?: string; + issuer?: string; + start_date_in_millis?: number; +} + +function getLicense(callCluster: LegacyAPICaller, local: boolean) { + return callCluster<{ license: ESLicense }>('transport.request', { + method: 'GET', + path: '/_license', + query: { + local, + // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. + accept_enterprise: 'true', + }, + }); +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 22dbd623930c5..177978c888ebe 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -64,6 +64,11 @@ import { registerTrustedAppsRoutes } from './endpoint/routes/trusted_apps'; import { securitySolutionSearchStrategyProvider } from './search_strategy/security_solution'; import { securitySolutionIndexFieldsProvider } from './search_strategy/index_fields'; import { securitySolutionTimelineSearchStrategyProvider } from './search_strategy/timeline'; +import { TelemetryEventsSender } from './lib/telemetry/sender'; +import { + TelemetryPluginStart, + TelemetryPluginSetup, +} from '../../../../src/plugins/telemetry/server'; export interface SetupPlugins { alerts: AlertingSetup; @@ -77,12 +82,14 @@ export interface SetupPlugins { spaces?: SpacesSetup; taskManager?: TaskManagerSetupContract; usageCollection?: UsageCollectionSetup; + telemetry?: TelemetryPluginSetup; } export interface StartPlugins { data: DataPluginStart; ingestManager?: IngestManagerStartContract; taskManager?: TaskManagerStartContract; + telemetry?: TelemetryPluginStart; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -107,6 +114,7 @@ export class Plugin implements IPlugin({ max: 3, maxAge: 1000 * 60 * 5 }); + this.telemetryEventsSender = new TelemetryEventsSender(this.logger); this.logger.debug('plugin initialized'); } @@ -241,6 +250,7 @@ export class Plugin implements IPlugin