From 5479505709d93cb9fb535ccc41328c3e997c14d7 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Mon, 11 Apr 2022 17:24:26 +0300 Subject: [PATCH 01/12] add transforms --- .../benchmark_score_transform.ts | 96 +++++++++++++++++++ .../create_transforms/create_transforms.ts | 38 ++++++++ .../latest_findings_transform.ts | 39 ++++++++ .../cloud_security_posture/server/plugin.ts | 6 +- 4 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts create mode 100644 x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts create mode 100644 x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts new file mode 100644 index 0000000000000..9bd55c04619df --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + LATEST_FINDINGS_INDEX_PATTERN, + BENCHMARK_SCORE_INDEX_PATTERN, +} from '../../common/constants'; + +export const benchmarkScoreTransform: TransformPutTransformRequest = { + transform_id: 'cis_kubernetes_benchmark.score-default-0.0.1', + description: 'Calculate latest findings score', + source: { + index: LATEST_FINDINGS_INDEX_PATTERN, + }, + dest: { + index: BENCHMARK_SCORE_INDEX_PATTERN, + }, + frequency: '30m', + sync: { + time: { + field: 'event.ingested', + delay: '60s', + }, + }, + retention_policy: { + time: { + field: '@timestamp', + max_age: '30d', + }, + }, + pivot: { + group_by: { + '@timestamp': { + date_histogram: { + field: '@timestamp', + calendar_interval: '1m', + }, + }, + }, + aggregations: { + total_findings: { + value_count: { + field: 'result.evaluation.keyword', + }, + }, + passed_findings: { + filter: { + term: { + 'result.evaluation.keyword': 'passed', + }, + }, + }, + failed_findings: { + filter: { + term: { + 'result.evaluation.keyword': 'failed', + }, + }, + }, + score_by_cluster_id: { + terms: { + field: 'cluster_id.keyword', + }, + aggregations: { + total_findings: { + value_count: { + field: 'result.evaluation.keyword', + }, + }, + passed_findings: { + filter: { + term: { + 'result.evaluation.keyword': 'passed', + }, + }, + }, + failed_findings: { + filter: { + term: { + 'result.evaluation.keyword': 'failed', + }, + }, + }, + }, + }, + }, + }, + _meta: { + managed: 'true', + }, +}; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts new file mode 100644 index 0000000000000..92a0d5d8e9f84 --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { transformError } from '@kbn/securitysolution-es-utils'; +import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; +import { latestFindingsTransform } from './latest_findings_transform'; +import { benchmarkScoreTransform } from './benchmark_score_transform'; + +// TODO: Move transforms to integration package +export const initializeCspTransforms = async (esClient: ElasticsearchClient, logger: Logger) => { + createTransformIfNotExists(esClient, latestFindingsTransform, logger); + createTransformIfNotExists(esClient, benchmarkScoreTransform, logger); +}; + +export const createTransformIfNotExists = async ( + esClient: ElasticsearchClient, + transform: TransformPutTransformRequest, + logger: Logger +) => { + try { + const isTransformExist = await esClient.transform.getTransform({ + transform_id: transform.transform_id, + allow_no_match: false, + }); + + if (!isTransformExist) { + await esClient.transform.putTransform(transform); + } + } catch (err) { + const error = transformError(err); + logger.error(`Failed to create transform ${transform.transform_id}`); + logger.error(error.message); + } +}; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts new file mode 100644 index 0000000000000..22ced95a5bd66 --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; +import { CSP_KUBEBEAT_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; + +export const latestFindingsTransform: TransformPutTransformRequest = { + transform_id: 'cis_kubernetes_benchmark.latest-default-0.0.1', + description: 'Defines findings transformation to view only the latest finding per resource', + source: { + index: CSP_KUBEBEAT_INDEX_PATTERN, + }, + dest: { + index: LATEST_FINDINGS_INDEX_PATTERN, + }, + frequency: '5m', + sync: { + time: { + field: 'event.ingested', + delay: '60s', + }, + }, + retention_policy: { + time: { + field: '@timestamp', + max_age: '3d', + }, + }, + latest: { + sort: '@timestamp', + unique_key: ['resource_id.keyword', 'rule.name.keyword', 'agent.id.keyword'], + }, + _meta: { + managed: 'true', + }, +}; diff --git a/x-pack/plugins/cloud_security_posture/server/plugin.ts b/x-pack/plugins/cloud_security_posture/server/plugin.ts index 386eb2373ad63..3794fc9366e1c 100755 --- a/x-pack/plugins/cloud_security_posture/server/plugin.ts +++ b/x-pack/plugins/cloud_security_posture/server/plugin.ts @@ -25,6 +25,7 @@ import { cspRuleTemplateAssetType } from './saved_objects/csp_rule_template'; import { cspRuleAssetType } from './saved_objects/csp_rule_type'; import { initializeCspRules } from './saved_objects/initialize_rules'; import { initializeCspTransformsIndices } from './create_indices/create_transforms_indices'; +import { initializeCspTransforms } from './create_transforms/create_transforms'; export interface CspAppContext { logger: Logger; @@ -72,7 +73,10 @@ export class CspPlugin }); initializeCspRules(core.savedObjects.createInternalRepository()); - initializeCspTransformsIndices(core.elasticsearch.client.asInternalUser, this.logger); + initializeCspTransformsIndices(core.elasticsearch.client.asInternalUser, this.logger).then( + (_) => initializeCspTransforms(core.elasticsearch.client.asInternalUser, this.logger) + ); + return {}; } public stop() {} From 163b2e3bd89d73ec0d843b1c56c56b5a474c1b34 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 17 Apr 2022 13:15:40 +0000 Subject: [PATCH 02/12] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/create_transforms/create_transforms.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index 92a0d5d8e9f84..7c4b3ea120931 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; -import type { ElasticsearchClient, Logger } from 'kibana/server'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { latestFindingsTransform } from './latest_findings_transform'; import { benchmarkScoreTransform } from './benchmark_score_transform'; From 175743d95f8832ff635fd8e4bee2fb1799e7356b Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 17:19:03 +0300 Subject: [PATCH 03/12] catch 404 --- .../common/constants.ts | 1 + .../benchmark_score_transform.ts | 2 +- .../create_transforms/create_transforms.ts | 26 ++++++++++++------- .../latest_findings_transform.ts | 9 ++++--- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 6f916f4911b7d..bf187f957b209 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -17,6 +17,7 @@ export const BENCHMARK_SCORE_INDEX_NAME = 'cloud_security_posture.scores'; export const AGENT_LOGS_INDEX_PATTERN = '.logs-cis_kubernetes_benchmark.metadata*'; export const CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings*'; +export const PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings-*'; export const LATEST_FINDINGS_INDEX_PATTERN = 'logs-' + LATEST_FINDINGS_INDEX_NAME + '-default'; export const BENCHMARK_SCORE_INDEX_PATTERN = 'logs-' + BENCHMARK_SCORE_INDEX_NAME + '-default'; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts index 9bd55c04619df..8837fea0fa183 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/benchmark_score_transform.ts @@ -12,7 +12,7 @@ import { } from '../../common/constants'; export const benchmarkScoreTransform: TransformPutTransformRequest = { - transform_id: 'cis_kubernetes_benchmark.score-default-0.0.1', + transform_id: 'cloud_security_posture.score-default-0.0.1', description: 'Calculate latest findings score', source: { index: LATEST_FINDINGS_INDEX_PATTERN, diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index 7c4b3ea120931..1bec42e25a391 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -22,17 +22,25 @@ export const createTransformIfNotExists = async ( logger: Logger ) => { try { - const isTransformExist = await esClient.transform.getTransform({ + await esClient.transform.getTransform({ transform_id: transform.transform_id, - allow_no_match: false, + allow_no_match: true, }); - - if (!isTransformExist) { - await esClient.transform.putTransform(transform); + } catch (existErr) { + const existError = transformError(existErr); + if (existError.statusCode === 404) { + try { + await esClient.transform.putTransform(transform); + } catch (createErr) { + const createError = transformError(createErr); + logger.error( + `Failed to create transform ${transform.transform_id}: ${createError.message}` + ); + } + } else { + logger.error( + `Failed to check if transform ${transform.transform_id} exists: ${existError.message}` + ); } - } catch (err) { - const error = transformError(err); - logger.error(`Failed to create transform ${transform.transform_id}`); - logger.error(error.message); } }; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts index 22ced95a5bd66..7109045dd2549 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts @@ -5,13 +5,16 @@ * 2.0. */ import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; -import { CSP_KUBEBEAT_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; +import { + PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN, + LATEST_FINDINGS_INDEX_PATTERN, +} from '../../common/constants'; export const latestFindingsTransform: TransformPutTransformRequest = { - transform_id: 'cis_kubernetes_benchmark.latest-default-0.0.1', + transform_id: 'cloud_security_posture.latest-default-0.0.1', description: 'Defines findings transformation to view only the latest finding per resource', source: { - index: CSP_KUBEBEAT_INDEX_PATTERN, + index: PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN, }, dest: { index: LATEST_FINDINGS_INDEX_PATTERN, From 9f7e602b37ae0c7fa784fa9e1510829c40a59133 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 17:45:18 +0300 Subject: [PATCH 04/12] add hyphen --- x-pack/plugins/cloud_security_posture/common/constants.ts | 3 +-- .../server/create_transforms/latest_findings_transform.ts | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index bf187f957b209..54006e21d2ebf 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -16,8 +16,7 @@ export const LATEST_FINDINGS_INDEX_NAME = 'cloud_security_posture.findings_lates export const BENCHMARK_SCORE_INDEX_NAME = 'cloud_security_posture.scores'; export const AGENT_LOGS_INDEX_PATTERN = '.logs-cis_kubernetes_benchmark.metadata*'; -export const CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings*'; -export const PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings-*'; +export const CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings-*'; export const LATEST_FINDINGS_INDEX_PATTERN = 'logs-' + LATEST_FINDINGS_INDEX_NAME + '-default'; export const BENCHMARK_SCORE_INDEX_PATTERN = 'logs-' + BENCHMARK_SCORE_INDEX_NAME + '-default'; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts index 7109045dd2549..64bf96c30783f 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts @@ -5,16 +5,13 @@ * 2.0. */ import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; -import { - PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN, - LATEST_FINDINGS_INDEX_PATTERN, -} from '../../common/constants'; +import { CSP_KUBEBEAT_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; export const latestFindingsTransform: TransformPutTransformRequest = { transform_id: 'cloud_security_posture.latest-default-0.0.1', description: 'Defines findings transformation to view only the latest finding per resource', source: { - index: PRIVILEGED_CSP_KUBEBEAT_INDEX_PATTERN, + index: CSP_KUBEBEAT_INDEX_PATTERN, }, dest: { index: LATEST_FINDINGS_INDEX_PATTERN, From 81f7c6fcdb8b6239e0dc14047fe4d329694616e8 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 18:45:29 +0300 Subject: [PATCH 05/12] start transforms, promise.all --- .../create_transforms_indices.ts | 30 ++++++------ .../create_transforms/create_transforms.ts | 48 ++++++++++++++++++- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_transforms_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_transforms_indices.ts index 048c122103a9e..c30bf09a60e0f 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_transforms_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_transforms_indices.ts @@ -21,20 +21,22 @@ export const initializeCspTransformsIndices = async ( esClient: ElasticsearchClient, logger: Logger ) => { - createIndexIfNotExists( - esClient, - LATEST_FINDINGS_INDEX_NAME, - LATEST_FINDINGS_INDEX_PATTERN, - latestFindingsMapping, - logger - ); - createIndexIfNotExists( - esClient, - BENCHMARK_SCORE_INDEX_NAME, - BENCHMARK_SCORE_INDEX_PATTERN, - benchmarkScoreMapping, - logger - ); + return Promise.all([ + createIndexIfNotExists( + esClient, + LATEST_FINDINGS_INDEX_NAME, + LATEST_FINDINGS_INDEX_PATTERN, + latestFindingsMapping, + logger + ), + createIndexIfNotExists( + esClient, + BENCHMARK_SCORE_INDEX_NAME, + BENCHMARK_SCORE_INDEX_PATTERN, + benchmarkScoreMapping, + logger + ), + ]); }; export const createIndexIfNotExists = async ( diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index 1bec42e25a391..ccd437de855af 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -12,8 +12,20 @@ import { benchmarkScoreTransform } from './benchmark_score_transform'; // TODO: Move transforms to integration package export const initializeCspTransforms = async (esClient: ElasticsearchClient, logger: Logger) => { - createTransformIfNotExists(esClient, latestFindingsTransform, logger); - createTransformIfNotExists(esClient, benchmarkScoreTransform, logger); + return Promise.all([ + initializeTransform(esClient, latestFindingsTransform, logger), + initializeTransform(esClient, benchmarkScoreTransform, logger), + ]); +}; + +export const initializeTransform = async ( + esClient: ElasticsearchClient, + transform: TransformPutTransformRequest, + logger: Logger +) => { + return createTransformIfNotExists(esClient, transform, logger).then(() => + startTransformIfNotStarted(esClient, transform.transform_id, logger) + ); }; export const createTransformIfNotExists = async ( @@ -44,3 +56,35 @@ export const createTransformIfNotExists = async ( } } }; + +export const startTransformIfNotStarted = async ( + esClient: ElasticsearchClient, + transformId: string, + logger: Logger +) => { + try { + const transformStats = await esClient.transform.getTransformStats({ + transform_id: transformId, + }); + const fetchedTransformStats = transformStats.transforms[0]; + if (fetchedTransformStats.state === 'stopped') { + try { + return esClient.transform.startTransform({ transform_id: transformId }); + } catch (startErr) { + const startError = transformError(startErr); + logger.error(`Failed starting transform ${transformId}: ${startError.message}`); + } + } else if ( + fetchedTransformStats.state === 'stopping' || + fetchedTransformStats.state === 'aborting' || + fetchedTransformStats.state === 'failed' + ) { + logger.error( + `Not starting transform ${transformId} since it's state is: ${fetchedTransformStats.state}` + ); + } + } catch (statsErr) { + const statsError = transformError(statsErr); + logger.error(`Failed to check if transform ${transformId} is started: ${statsError.message}`); + } +}; From 0418617f4429d7e68686177d14c3c727e946d395 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 18:49:59 +0300 Subject: [PATCH 06/12] return is clearer --- .../server/create_transforms/create_transforms.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index ccd437de855af..b57d55178ccb8 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -42,7 +42,7 @@ export const createTransformIfNotExists = async ( const existError = transformError(existErr); if (existError.statusCode === 404) { try { - await esClient.transform.putTransform(transform); + return await esClient.transform.putTransform(transform); } catch (createErr) { const createError = transformError(createErr); logger.error( @@ -69,7 +69,7 @@ export const startTransformIfNotStarted = async ( const fetchedTransformStats = transformStats.transforms[0]; if (fetchedTransformStats.state === 'stopped') { try { - return esClient.transform.startTransform({ transform_id: transformId }); + return await esClient.transform.startTransform({ transform_id: transformId }); } catch (startErr) { const startError = transformError(startErr); logger.error(`Failed starting transform ${transformId}: ${startError.message}`); From 83e2948a43191687b4f4e488b261bdc81921c86f Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 20:35:01 +0300 Subject: [PATCH 07/12] add tests --- .../create_transforms.test.ts | 116 ++++++++++++++++++ .../create_transforms/create_transforms.ts | 5 +- 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts new file mode 100644 index 0000000000000..a04fa6df7b6f1 --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from '@kbn/core/server/elasticsearch/client/mocks'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { createTransformIfNotExists, startTransformIfNotStarted } from './create_transforms'; +import { latestFindingsTransform } from './latest_findings_transform'; + +const mockEsClient = elasticsearchClientMock.createClusterClient().asScoped().asInternalUser; + +describe('createTransformIfNotExist', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + jest.resetAllMocks(); + }); + + it('expect not to create if already exists', async () => { + mockEsClient.transform.getTransform.mockResolvedValue({ transforms: [], count: 1 }); + await createTransformIfNotExists(mockEsClient, latestFindingsTransform, logger); + expect(mockEsClient.transform.getTransform).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.getTransform).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + expect(mockEsClient.transform.putTransform).toHaveBeenCalledTimes(0); + }); + + it('expect to create if does not already exist', async () => { + mockEsClient.transform.getTransform.mockRejectedValue({ statusCode: 404 }); + await createTransformIfNotExists(mockEsClient, latestFindingsTransform, logger); + expect(mockEsClient.transform.getTransform).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.getTransform).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + expect(mockEsClient.transform.putTransform).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.putTransform).toHaveBeenCalledWith(latestFindingsTransform); + }); + + it('expect not to create if get error is not 404', async () => { + mockEsClient.transform.getTransform.mockRejectedValue({ statusCode: 400 }); + await createTransformIfNotExists(mockEsClient, latestFindingsTransform, logger); + expect(mockEsClient.transform.getTransform).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.putTransform).toHaveBeenCalledTimes(0); + }); +}); + +function getTransformWithState(state: string) { + return { + state, + checkpointing: { last: { checkpoint: 1 } }, + id: '', + stats: { + documents_indexed: 0, + documents_processed: 0, + exponential_avg_checkpoint_duration_ms: 0, + exponential_avg_documents_indexed: 0, + exponential_avg_documents_processed: 0, + index_failures: 0, + index_time_in_ms: 0, + index_total: 0, + pages_processed: 0, + processing_time_in_ms: 0, + processing_total: 0, + search_failures: 0, + search_time_in_ms: 0, + search_total: 0, + trigger_count: 0, + }, + }; +} + +describe('startTransformIfNotStarted', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + jest.resetAllMocks(); + }); + + ['failed', 'stopping', 'started', 'aborting', 'indexing'].forEach((state) => + it(`expect not to start if state is ${state}`, async () => { + mockEsClient.transform.getTransformStats.mockResolvedValue({ + transforms: [getTransformWithState(state)], + count: 1, + }); + await startTransformIfNotStarted(mockEsClient, latestFindingsTransform.transform_id, logger); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + expect(mockEsClient.transform.startTransform).toHaveBeenCalledTimes(0); + }) + ); + + it('expect to start if state is stopped', async () => { + mockEsClient.transform.getTransformStats.mockResolvedValue({ + transforms: [getTransformWithState('stopped')], + count: 1, + }); + await startTransformIfNotStarted(mockEsClient, latestFindingsTransform.transform_id, logger); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + expect(mockEsClient.transform.startTransform).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.startTransform).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + }); +}); diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index b57d55178ccb8..982e88e9848d6 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -36,7 +36,6 @@ export const createTransformIfNotExists = async ( try { await esClient.transform.getTransform({ transform_id: transform.transform_id, - allow_no_match: true, }); } catch (existErr) { const existError = transformError(existErr); @@ -66,6 +65,10 @@ export const startTransformIfNotStarted = async ( const transformStats = await esClient.transform.getTransformStats({ transform_id: transformId, }); + if (transformStats.count <= 0) { + logger.error(`Failed starting transform ${transformId}: couldn't find transform`); + return; + } const fetchedTransformStats = transformStats.transforms[0]; if (fetchedTransformStats.state === 'stopped') { try { From 378db455aa0aaa0d88672759d20ab99da28183a2 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Sun, 17 Apr 2022 20:43:12 +0300 Subject: [PATCH 08/12] transform rename --- .../server/create_transforms/latest_findings_transform.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts index 64bf96c30783f..020421e793df9 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts @@ -8,7 +8,7 @@ import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/ap import { CSP_KUBEBEAT_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; export const latestFindingsTransform: TransformPutTransformRequest = { - transform_id: 'cloud_security_posture.latest-default-0.0.1', + transform_id: 'cloud_security_posture.findings_latest-default-0.0.1', description: 'Defines findings transformation to view only the latest finding per resource', source: { index: CSP_KUBEBEAT_INDEX_PATTERN, From 8fb23cca3e7c484d26a60e34e348eb4755ab2494 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Mon, 18 Apr 2022 10:36:38 +0300 Subject: [PATCH 09/12] add test --- .../create_transforms/create_transforms.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts index a04fa6df7b6f1..65a4507de2511 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.test.ts @@ -98,6 +98,19 @@ describe('startTransformIfNotStarted', () => { }) ); + it('expect not to start if transform not found', async () => { + mockEsClient.transform.getTransformStats.mockResolvedValue({ + transforms: [], + count: 0, + }); + await startTransformIfNotStarted(mockEsClient, latestFindingsTransform.transform_id, logger); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledTimes(1); + expect(mockEsClient.transform.getTransformStats).toHaveBeenCalledWith({ + transform_id: latestFindingsTransform.transform_id, + }); + expect(mockEsClient.transform.startTransform).toHaveBeenCalledTimes(0); + }); + it('expect to start if state is stopped', async () => { mockEsClient.transform.getTransformStats.mockResolvedValue({ transforms: [getTransformWithState('stopped')], From 3ca2fae13994bbea2e085399563d97d709385fe2 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Mon, 18 Apr 2022 14:53:37 +0300 Subject: [PATCH 10/12] use exact pattern --- x-pack/plugins/cloud_security_posture/common/constants.ts | 4 ++++ .../server/create_transforms/latest_findings_transform.ts | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 9c2aee585e307..a45222ebbcec9 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -13,11 +13,15 @@ export const UPDATE_RULES_CONFIG_ROUTE_PATH = export const CSP_FINDINGS_INDEX_NAME = 'findings'; export const CIS_KUBERNETES_PACKAGE_NAME = 'cis_kubernetes_benchmark'; +export const FINDINGS_DATA_STREAM_NAME = + // Currently 'cis_kubernetes_benchmark.findings', To be refactored to 'cloud_security_posture.findings' + CIS_KUBERNETES_PACKAGE_NAME + '.' + CSP_FINDINGS_INDEX_NAME; export const LATEST_FINDINGS_INDEX_NAME = 'cloud_security_posture.findings_latest'; export const BENCHMARK_SCORE_INDEX_NAME = 'cloud_security_posture.scores'; export const AGENT_LOGS_INDEX_PATTERN = '.logs-cis_kubernetes_benchmark.metadata*'; export const CSP_KUBEBEAT_INDEX_PATTERN = 'logs-cis_kubernetes_benchmark.findings-*'; +export const FINDINGS_INDEX_PATTERN = 'logs-' + FINDINGS_DATA_STREAM_NAME + '-default'; export const LATEST_FINDINGS_INDEX_PATTERN = 'logs-' + LATEST_FINDINGS_INDEX_NAME + '-default'; export const BENCHMARK_SCORE_INDEX_PATTERN = 'logs-' + BENCHMARK_SCORE_INDEX_NAME + '-default'; diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts index 020421e793df9..fc042149f3193 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/latest_findings_transform.ts @@ -5,13 +5,13 @@ * 2.0. */ import type { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; -import { CSP_KUBEBEAT_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; +import { FINDINGS_INDEX_PATTERN, LATEST_FINDINGS_INDEX_PATTERN } from '../../common/constants'; export const latestFindingsTransform: TransformPutTransformRequest = { transform_id: 'cloud_security_posture.findings_latest-default-0.0.1', description: 'Defines findings transformation to view only the latest finding per resource', source: { - index: CSP_KUBEBEAT_INDEX_PATTERN, + index: FINDINGS_INDEX_PATTERN, }, dest: { index: LATEST_FINDINGS_INDEX_PATTERN, From a4017cc84052cd8f8c5c65bc888e2c1e13d2bc0d Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Mon, 18 Apr 2022 15:26:14 +0300 Subject: [PATCH 11/12] only start if created --- .../server/create_transforms/create_transforms.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index 982e88e9848d6..42794f0dd43b9 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -23,9 +23,11 @@ export const initializeTransform = async ( transform: TransformPutTransformRequest, logger: Logger ) => { - return createTransformIfNotExists(esClient, transform, logger).then(() => - startTransformIfNotStarted(esClient, transform.transform_id, logger) - ); + return createTransformIfNotExists(esClient, transform, logger).then((succeeded) => { + if (succeeded) { + startTransformIfNotStarted(esClient, transform.transform_id, logger); + } + }); }; export const createTransformIfNotExists = async ( @@ -37,11 +39,13 @@ export const createTransformIfNotExists = async ( await esClient.transform.getTransform({ transform_id: transform.transform_id, }); + return true; } catch (existErr) { const existError = transformError(existErr); if (existError.statusCode === 404) { try { - return await esClient.transform.putTransform(transform); + await esClient.transform.putTransform(transform); + return true; } catch (createErr) { const createError = transformError(createErr); logger.error( @@ -54,6 +58,7 @@ export const createTransformIfNotExists = async ( ); } } + return false; }; export const startTransformIfNotStarted = async ( From da2c8948b0b33dbec676f94f71388ddac00d05f4 Mon Sep 17 00:00:00 2001 From: Eyal Kraft Date: Mon, 18 Apr 2022 15:46:49 +0300 Subject: [PATCH 12/12] add function doc --- .../server/create_transforms/create_transforms.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts index 42794f0dd43b9..3347d5f36b5d8 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_transforms/create_transforms.ts @@ -30,6 +30,13 @@ export const initializeTransform = async ( }); }; +/** + * Checks if a transform exists, And if not creates it + * + * @param transform - the transform to create. If a transform with the same transform_id already exists, nothing is created or updated. + * + * @return true if the transform exits or created, false otherwise. + */ export const createTransformIfNotExists = async ( esClient: ElasticsearchClient, transform: TransformPutTransformRequest,