Merge branch 'main' into revert-164574-task-manager/force-task-state-…

…validation
elastic · Oct 3, 2023 · 6e8eb59 · 6e8eb59
2 parents b3bf72a + efdd573
commit 6e8eb59
Show file tree

Hide file tree

Showing 12 changed files with 274 additions and 56 deletions.
diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -50,7 +50,7 @@ steps:
         agents:
           queue: n2-4-spot
         depends_on: build
-        timeout_in_minutes: 40
+        timeout_in_minutes: 60
         parallelism: 2
         retry:
           automatic:
@@ -74,8 +74,8 @@ steps:
         agents:
           queue: n2-4-spot
         depends_on: build
-        timeout_in_minutes: 120
-        parallelism: 2
+        timeout_in_minutes: 60
+        parallelism: 4
         retry:
           automatic:
             - exit_status: '*'

diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -84,7 +84,7 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 40
+    timeout_in_minutes: 60
     parallelism: 2
     retry:
       automatic:
@@ -108,8 +108,8 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
-    parallelism: 2
+    timeout_in_minutes: 60
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -62,49 +62,49 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 40
+    timeout_in_minutes: 60
     parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  # status_exception: Native role management is not enabled in this Elasticsearch instance
-  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
-  #   label: 'Serverless Security Defend Workflows Cypress Tests'
-  #   agents:
-  #     queue: n2-4-spot
-  #   depends_on: build
-  #   timeout_in_minutes: 40
-  #   retry:
-  #     automatic:
-  #       - exit_status: '*'
-  #         limit: 1
-
-  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
-    label: 'Serverless Security Investigations Cypress Tests'
+  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
+    label: 'Serverless Explore - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 40
+    timeout_in_minutes: 60
     parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
-    label: 'Serverless Security Explore Cypress Tests'
+  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
+    label: 'Serverless Investigations - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 40
-    parallelism: 2
+    timeout_in_minutes: 60
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
+  # status_exception: Native role management is not enabled in this Elasticsearch instance
+  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
+  #   label: 'Serverless Security Defend Workflows Cypress Tests'
+  #   agents:
+  #     queue: n2-4-spot
+  #   depends_on: build
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '*'
+  #         limit: 1
+
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

diff --git a/package.json b/package.json
@@ -1564,7 +1564,7 @@
     "pixelmatch": "^5.3.0",
     "playwright": "=1.37.0",
     "pngjs": "^3.4.0",
-    "postcss": "^8.4.14",
+    "postcss": "^8.4.31",
     "postcss-loader": "^4.2.0",
     "postcss-prefix-selector": "^1.16.0",
     "postcss-scss": "^4.0.4",

diff --git a/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts
@@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "f11cc19275f4c3e4ee7c5cd6423b6706b21b989d",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
         "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0",
-        "ingest-package-policies": "8ec637429836f80f1fcc798bcee7c5916eceaed5",
+        "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea",
         "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
         "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",

diff --git a/x-pack/plugins/cloud_security_posture/README.md b/x-pack/plugins/cloud_security_posture/README.md
@@ -6,17 +6,21 @@ Cloud Posture automates the identification and remediation of risks across cloud
 
 ## Development
 
-read [Kibana Contributing Guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for more details
+Read [Kibana Contributing Guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for more details
 
 ## Testing
 
-for general guidelines, read [Kibana Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html) for more details
+For general guidelines, read [Kibana Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html) for more details
 
 ### Tests
 
 1. Unit Tests (Jest) - located in sibling files to the source code
-2. [Integration Tests](../../test/api_integration/apis/cloud_security_posture/index.ts)
-3. [End-to-End Tests](../../test/cloud_security_posture_functional/pages/index.ts)
+1. [API Integration Tests](../../test/api_integration/apis/cloud_security_posture/config.ts)
+1. [Telemetry Integration Tests](../../test/cloud_security_posture_api/config.ts)
+1. [End-to-End Tests](../../test/cloud_security_posture_functional/config.ts)
+1. [Serverless API Integration tests](../../test_serverless/api_integration/test_suites/security/config.ts)
+1. [Serverless End-to-End Tests](../../test_serverless/functional/test_suites/security/config.ts)
+
 
 ### Tools
 
@@ -32,13 +36,25 @@ Run **ESLint**:
 yarn lint:es x-pack/plugins/cloud_security_posture
 ```
 
+Run **i18n check**:
+```bash
+node scripts/i18n_check.js
+```
+
+> **Note**
+>
+> i18n should run on project scope as it checks translations files outside of our plugin.
+>
+> Fixes can be applied using the --fix flag
+
 Run [**Unit Tests**](https://www.elastic.co/guide/en/kibana/current/development-tests.html#_unit_testing):
 
 ```bash
 yarn test:jest --config x-pack/plugins/cloud_security_posture/jest.config.js
 ```
 
 > **Note**
+>
 > for a coverage report, add the `--coverage` flag, and run `open target/kibana-coverage/jest/x-pack/plugins/cloud_security_posture/index.html`
 
 Run [**Integration Tests**](https://docs.elastic.dev/kibana-dev-docs/tutorials/testing-plugins#):
@@ -50,14 +66,45 @@ yarn test:ftr --config x-pack/test/api_integration/config.ts
 Run [**End-to-End Tests**](https://www.elastic.co/guide/en/kibana/current/development-tests.html#_running_functional_tests):
 
 ```bash
-yarn test:ftr --config x-pack/test/cloud_security_posture_functional/config.ts --debug
+yarn test:ftr --config x-pack/test/cloud_security_posture_functional/config.ts
+yarn test:ftr --config x-pack/test/api_integration/config.ts --include-tag=cloud_security_posture
+yarn test:ftr --config x-pack/test/cloud_security_posture_api/config.ts
+yarn test:ftr --config x-pack/test_serverless/api_integration/test_suites/security/config.ts --include-tag=cloud_security_posture
+yarn test:ftr --config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
 ```
 
-<br/>
+#### Run **FTR tests (integration or e2e) for development**
+
+Functional test runner (FTR) can be used separately with `ftr:runner` and `ftr:server`. This is convenient while developing tests.
 
-test runner (FTR) can be used separately with `ftr:runner` and `ftr:server`:
+For example, 
 
+run ESS (stateful) api integration tests:
 ```bash
 yarn test:ftr:server --config x-pack/test/api_integration/config.ts
-yarn test:ftr:runner --include-tag=cloud_security_posture --config x-pack/test/api_integration/config.ts
+yarn test:ftr:runner --config x-pack/test/api_integration/apis/cloud_security_posture/config.ts
+```
+
+run ESS (stateful) telemetry integration tests:
+```bash
+yarn test:ftr:server --config x-pack/test/cloud_security_posture_api/config.ts
+yarn test:ftr:runner --config x-pack/test/cloud_security_posture_api/config.ts
 ```
+
+run ESS (stateful) e2e tests:
+```bash
+yarn test:ftr:server --config x-pack/test/cloud_security_posture_functional/config.ts
+yarn test:ftr:runner --config x-pack/test/cloud_security_posture_functional/config.ts
+```
+
+run serverless api integration tests:
+```bash
+yarn test:ftr:server --config x-pack/test_serverless/api_integration/test_suites/security/config.ts
+yarn test:ftr:runner --config x-pack/test_serverless/api_integration/test_suites/security/config.ts --include-tag=cloud_security_posture
+```
+
+run serverless e2e tests:
+```bash
+yarn test:ftr:server --config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
+yarn test:ftr:runner ---config x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
+```
diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -30,17 +30,14 @@ import {
   migratePackagePolicyToV8110,
 } from './migrations/security_solution/to_v8_11_0';
 
+import { migrateCspPackagePolicyToV8110 } from './migrations/cloud_security_posture';
+
 import { migrateOutputEvictionsFromV8100, migrateOutputToV8100 } from './migrations/to_v8_10_0';
 
 import { migrateSyntheticsPackagePolicyToV8100 } from './migrations/synthetics/to_v8_10_0';
 
 import { migratePackagePolicyEvictionsFromV8100 } from './migrations/security_solution/to_v8_10_0';
 
-import {
-  migratePackagePolicyEvictionsFromV81102,
-  migratePackagePolicyToV81102,
-} from './migrations/security_solution/to_v8_11_0_2';
-
 import {
   migrateAgentPolicyToV7100,
   migratePackagePolicyToV7100,
@@ -78,6 +75,10 @@ import {
 } from './migrations/security_solution';
 import { migratePackagePolicyToV880 } from './migrations/to_v8_8_0';
 import { migrateAgentPolicyToV890 } from './migrations/to_v8_9_0';
+import {
+  migratePackagePolicyToV81102,
+  migratePackagePolicyEvictionsFromV81102,
+} from './migrations/security_solution/to_v8_11_0_2';
 
 /*
  * Saved object types and mappings
@@ -351,6 +352,14 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
           forwardCompatibility: migratePackagePolicyEvictionsFromV81102,
         },
       },
+      '4': {
+        changes: [
+          {
+            type: 'data_backfill',
+            backfillFn: migrateCspPackagePolicyToV8110,
+          },
+        ],
+      },
     },
     migrations: {
       '7.10.0': migratePackagePolicyToV7100,

diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/index.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/cloud_security_posture/index.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { migrateCspPackagePolicyToV8110 } from './to_v8_11_0';