Merge branch 'main' into feat/slo-transforms-remove-transform-and-ing…

…est-pipeline-privilege-requirements

.buildkite/package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.10.0",
-    "axios": "^1.6.3",
+    "axios": "^1.7.4",
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",

.buildkite/pipelines/on_merge.yml

@@ -37,7 +37,7 @@ steps:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod
       provider: gcp
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60
@@ -589,6 +589,11 @@ steps:
       preemptible: true
     artifact_paths:
       "target/plugin_so_types_snapshot.json"
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/pull_request/base.yml

@@ -23,7 +23,7 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60

.buildkite/scripts/common/util.sh

@@ -12,7 +12,7 @@ is_pr_with_label() {
 
   IFS=',' read -ra labels <<< "${GITHUB_PR_LABELS:-}"
 
-  for label in "${labels[@]}"
+  for label in "${labels[@]:-}"
   do
     if [ "$label" == "$match" ]; then
       return

.buildkite/scripts/steps/checks/event_log.sh

@@ -8,7 +8,25 @@ echo --- Check Event Log Schema
 
 # event log schema is pinned to a specific version of ECS
 ECS_STABLE_VERSION=1.8
-git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+
+# we can potentially skip this check on a local env, if ../ecs is present, and modified by the developer
+if [[ "${CI:-false}" =~ ^(0|false)$ ]] && [[ -d '../ecs' ]]; then
+  LOCAL_ECS_BRANCH=$(git -C ../ecs branch --show-current)
+  if [[ "$LOCAL_ECS_BRANCH" != "$ECS_STABLE_VERSION" ]]; then
+    echo "Skipping event log schema check because ECS schema is not on $ECS_STABLE_VERSION."
+    exit 0
+  fi
+
+  TOUCHED_FILES=$(git -C ../ecs status --porcelain)
+  if [[ -n "$TOUCHED_FILES" ]]; then
+    echo "Skipping event log schema check because ECS schema files have been modified."
+    exit 0
+  fi
+
+  echo "../ecs is already cloned and @ $ECS_STABLE_VERSION"
+else
+  git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+fi
 
 node x-pack/plugins/event_log/scripts/create_schemas.js
 

.buildkite/scripts/steps/checks/quick_checks.txt

@@ -0,0 +1,19 @@
+.buildkite/scripts/steps/checks/precommit_hook.sh
+.buildkite/scripts/steps/checks/ts_projects.sh
+.buildkite/scripts/steps/checks/packages.sh
+.buildkite/scripts/steps/checks/bazel_packages.sh
+.buildkite/scripts/steps/checks/verify_notice.sh
+.buildkite/scripts/steps/checks/plugin_list_docs.sh
+.buildkite/scripts/steps/checks/event_log.sh
+.buildkite/scripts/steps/checks/telemetry.sh
+.buildkite/scripts/steps/checks/jest_configs.sh
+.buildkite/scripts/steps/checks/bundle_limits.sh
+.buildkite/scripts/steps/checks/i18n.sh
+.buildkite/scripts/steps/checks/file_casing.sh
+.buildkite/scripts/steps/checks/licenses.sh
+.buildkite/scripts/steps/checks/test_projects.sh
+.buildkite/scripts/steps/checks/test_hardening.sh
+.buildkite/scripts/steps/checks/ftr_configs.sh
+.buildkite/scripts/steps/checks/yarn_deduplicate.sh
+.buildkite/scripts/steps/checks/prettier_topology.sh
+.buildkite/scripts/steps/checks/renovate.sh

.buildkite/scripts/steps/quick_checks.sh

@@ -2,25 +2,9 @@
 
 set -euo pipefail
 
-export DISABLE_BOOTSTRAP_VALIDATION=false
-.buildkite/scripts/bootstrap.sh
+if [[ "${CI:-}" =~ ^(1|true)$ ]]; then
+  export DISABLE_BOOTSTRAP_VALIDATION=false
+  .buildkite/scripts/bootstrap.sh
+fi
 
-.buildkite/scripts/steps/checks/precommit_hook.sh
-.buildkite/scripts/steps/checks/ts_projects.sh
-.buildkite/scripts/steps/checks/packages.sh
-.buildkite/scripts/steps/checks/bazel_packages.sh
-.buildkite/scripts/steps/checks/verify_notice.sh
-.buildkite/scripts/steps/checks/plugin_list_docs.sh
-.buildkite/scripts/steps/checks/event_log.sh
-.buildkite/scripts/steps/checks/telemetry.sh
-.buildkite/scripts/steps/checks/jest_configs.sh
-.buildkite/scripts/steps/checks/bundle_limits.sh
-.buildkite/scripts/steps/checks/i18n.sh
-.buildkite/scripts/steps/checks/file_casing.sh
-.buildkite/scripts/steps/checks/licenses.sh
-.buildkite/scripts/steps/checks/test_projects.sh
-.buildkite/scripts/steps/checks/test_hardening.sh
-.buildkite/scripts/steps/checks/ftr_configs.sh
-.buildkite/scripts/steps/checks/yarn_deduplicate.sh
-.buildkite/scripts/steps/checks/prettier_topology.sh
-.buildkite/scripts/steps/checks/renovate.sh
+node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.txt

.github/CODEOWNERS

@@ -508,7 +508,7 @@ x-pack/packages/index-management @elastic/kibana-management
 x-pack/plugins/index_management @elastic/kibana-management
 test/plugin_functional/plugins/index_patterns @elastic/kibana-data-discovery
 x-pack/packages/ml/inference_integration_flyout @elastic/ml-ui
-x-pack/plugins/inference @elastic/kibana-core
+x-pack/plugins/inference @elastic/appex-ai-infra
 x-pack/packages/kbn-infra-forge @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team
 x-pack/plugins/ingest_pipelines @elastic/kibana-management
@@ -733,6 +733,7 @@ src/plugins/screenshot_mode @elastic/appex-sharedux
 x-pack/examples/screenshotting_example @elastic/appex-sharedux
 x-pack/plugins/screenshotting @elastic/kibana-reporting-services
 packages/kbn-search-api-panels @elastic/search-kibana
+x-pack/plugins/search_assistant @elastic/search-kibana
 packages/kbn-search-connectors @elastic/search-kibana
 x-pack/plugins/search_connectors @elastic/search-kibana
 packages/kbn-search-errors @elastic/kibana-data-discovery
@@ -747,12 +748,14 @@ packages/kbn-search-types @elastic/kibana-data-discovery
 x-pack/plugins/searchprofiler @elastic/kibana-management
 x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security
 x-pack/packages/security/api_key_management @elastic/kibana-security
+x-pack/packages/security/authorization_core @elastic/kibana-security
 x-pack/packages/security/form_components @elastic/kibana-security
 packages/kbn-security-hardening @elastic/kibana-security
 x-pack/plugins/security @elastic/kibana-security
 x-pack/packages/security/plugin_types_common @elastic/kibana-security
 x-pack/packages/security/plugin_types_public @elastic/kibana-security
 x-pack/packages/security/plugin_types_server @elastic/kibana-security
+x-pack/packages/security/role_management_model @elastic/kibana-security
 x-pack/packages/security-solution/distribution_bar @elastic/kibana-cloud-security-posture
 x-pack/plugins/security_solution_ess @elastic/security-solution
 x-pack/packages/security-solution/features @elastic/security-threat-hunting-explore
@@ -1306,7 +1309,6 @@ x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor te
 /x-pack/test_serverless/**/test_suites/common/saved_objects_management/ @elastic/kibana-core
 /x-pack/test_serverless/api_integration/test_suites/common/core/ @elastic/kibana-core
 /x-pack/test_serverless/api_integration/test_suites/**/telemetry/ @elastic/kibana-core
-/x-pack/plugins/inference @elastic/kibana-core @elastic/obs-ai-assistant @elastic/security-generative-ai
 #CC# /src/core/server/csp/ @elastic/kibana-core
 #CC# /src/plugins/saved_objects/ @elastic/kibana-core
 #CC# /x-pack/plugins/cloud/ @elastic/kibana-core
@@ -1315,6 +1317,9 @@ x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor te
 #CC# /src/plugins/newsfeed @elastic/kibana-core
 #CC# /x-pack/plugins/global_search_providers/ @elastic/kibana-core
 
+# AppEx AI Infra
+/x-pack/plugins/inference @elastic/appex-ai-infra @elastic/obs-ai-assistant @elastic/security-generative-ai
+
 # AppEx Platform Services Security
 x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts  @elastic/kibana-security
 

docs/developer/plugin-list.asciidoc

@@ -795,6 +795,10 @@ Elastic.
 It uses Chromium and Puppeteer underneath to run the browser in headless mode.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/search_assistant/README.md[searchAssistant]
+|This holds the Search AI Assistant which targets Search users and Serverless Elasticsearch.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/search_connectors/README.mdx[searchConnectors]
 |This plugin contains common assets and endpoints for the use of connectors in Kibana. Primarily used by the enterprise_search and serverless_search plugins.
 

docs/management/connectors/pre-configured-connectors.asciidoc

@@ -513,14 +513,19 @@ xpack.actions.preconfigured:
       jwtKeyId: fedcbazyxwvutsrqponmlkjihgfedcba <4>
     secrets:
       clientSecret: secretsecret <5>
-      privateKey: -----BEGIN RSA PRIVATE KEY-----\nprivatekeyhere\n-----END RSA PRIVATE KEY----- <6>
+      privateKey: |  <6>
+        -----BEGIN RSA PRIVATE KEY-----
+        MIIE...
+        KAgD...
+        ... multiple lines of key data ...
+        -----END RSA PRIVATE KEY----- 
 --
 <1> Specifies whether the connector uses basic or OAuth authentication.
 <2> The user identifier.
 <3> The client identifier assigned to your OAuth application.
 <4> The key identifier assigned to the JWT verifier map of your OAuth application.
 <5> The client secret assigned to your OAuth application.
-<6> The RSA private key. If it has a password, you must also provide `privateKeyPassword`.
+<6> The RSA private key in multiline format. If it has a password, you must also provide `privateKeyPassword`.
 
 [float]
 [[preconfigured-servicenow-configuration]]
@@ -563,14 +568,19 @@ xpack.actions.preconfigured:
       jwtKeyId: fedcbazyxwvutsrqponmlkjihgfedcba <4>
     secrets:
       clientSecret: secretsecret <5>
-      privateKey: -----BEGIN RSA PRIVATE KEY-----\nprivatekeyhere\n-----END RSA PRIVATE KEY----- <6>
+      privateKey: | <6>
+        -----BEGIN RSA PRIVATE KEY-----
+        MIIE...
+        KAgD...
+        ... multiple lines of key data ...
+        -----END RSA PRIVATE KEY----- 
 --
 <1> Specifies whether the connector uses basic or OAuth authentication.
 <2> The user identifier.
 <3> The client identifier assigned to your OAuth application.
 <4> The key ID assigned to the JWT verifier map of your OAuth application.
 <5> The client secret assigned to the OAuth application.
-<6> The RSA private key. If it has a password, you must also provide `privateKeyPassword`.
+<6> The RSA private key in multiline format. If it has a password, you must also provide `privateKeyPassword`.
 
 [float]
 [[preconfigured-servicenow-sir-configuration]]
@@ -613,14 +623,19 @@ xpack.actions.preconfigured:
       jwtKeyId: fedcbazyxwvutsrqponmlkjihgfedcba <4>
     secrets:
       clientSecret: secretsecret <5>
-      privateKey: -----BEGIN RSA PRIVATE KEY-----\nprivatekeyhere\n-----END RSA PRIVATE KEY----- <6>
+      privateKey: | <6>
+        -----BEGIN RSA PRIVATE KEY-----
+        MIIE...
+        KAgD...
+        ... multiple lines of key data ...
+        -----END RSA PRIVATE KEY----- 
 --
 <1> Specifies whether the connector uses basic or OAuth authentication.
 <2> The user identifier.
 <3> The client identifier assigned to the OAuth application.
 <4> The key ID assigned to the JWT verifier map of your OAuth application.
 <5> The client secret assigned to the OAuth application.
-<6> The RSA private key. If it has a password, you must also specify
+<6> The RSA private key in multiline format. If it has a password, you must also specify
 `privateKeyPassword`.
 
 

docs/settings/alert-action-settings.asciidoc

@@ -67,7 +67,9 @@ xpack.actions.customHostSettings:
       certificateAuthoritiesFiles: [ 'one.crt' ]
       certificateAuthoritiesData: |
           -----BEGIN CERTIFICATE-----
-          ... multiple lines of certificate data here ...
+          MIIDTD...
+          CwUAMD...
+          ... multiple lines of certificate data ...
           -----END CERTIFICATE-----
     smtp:
       requireTLS: true
@@ -125,9 +127,8 @@ A file name or list of file names of PEM-encoded certificate files to use
 to validate the server.
 
 `xpack.actions.customHostSettings[n].ssl.certificateAuthoritiesData` {ess-icon}::
-The contents of a PEM-encoded certificate file, or multiple files appended
-into a single string.  This configuration can be used for environments where
-the files cannot be made available.
+The contents of one or more PEM-encoded certificate files in multiline format.
+This configuration can be used for environments where the files cannot be made available.
 
 [[action-config-email-domain-allowlist]] `xpack.actions.email.domain_allowlist`  {ess-icon}::
 A list of allowed email domains which can be used with the email connector. When this setting is not used, all email domains are allowed. When this setting is used, if any email is attempted to be sent that (a) includes an addressee with an email domain that is not in the allowlist, or (b) includes a from address domain that is not in the allowlist, it will fail with a message indicating the email is not allowed.

docs/settings/apm-settings.asciidoc

@@ -82,19 +82,19 @@ Sets a `fixed_interval` for date histograms in metrics aggregations. Defaults to
 Set to `false` to disable cloud APM migrations. Defaults to `true`.
 
 `xpack.apm.indices.error` {ess-icon}::
-Matcher for all error indices. Defaults to `logs-apm*,apm-*`.
+Matcher for all error indices. Defaults to `logs-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.onboarding` {ess-icon}::
 Matcher for all onboarding indices. Defaults to `apm-*`.
 
 `xpack.apm.indices.span` {ess-icon}::
-Matcher for all span indices. Defaults to `traces-apm*,apm-*`.
+Matcher for all span indices. Defaults to `traces-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.transaction` {ess-icon}::
-Matcher for all transaction indices. Defaults to `traces-apm*,apm-*`.
+Matcher for all transaction indices. Defaults to `traces-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.metric` {ess-icon}::
-Matcher for all metrics indices. Defaults to `metrics-apm*,apm-*`.
+Matcher for all metrics indices. Defaults to `metrics-apm*,apm-*,metrics-*.otel-*`.
 
 `xpack.apm.indices.sourcemap` {ess-icon}::
 Matcher for all source map indices. Defaults to `apm-*`.