Merge branch 'main' into task/olm-3092-trusted-app-post-port-cleanup

.bazelversion

@@ -1 +1 @@
-5.2.0
+5.1.1

.buildkite/ftr_configs.yml

@@ -73,6 +73,7 @@ enabled:
   - test/functional/apps/discover/config.ts
   - test/functional/apps/getting_started/config.ts
   - test/functional/apps/home/config.ts
+  - test/functional/apps/kibana_overview/config.ts
   - test/functional/apps/management/config.ts
   - test/functional/apps/saved_objects_management/config.ts
   - test/functional/apps/status_page/config.ts

.buildkite/pipelines/build_api_docs.yml

@@ -0,0 +1,13 @@
+env:
+  PUBLISH_API_DOCS_CHANGES: 'true'
+steps:
+  - command: .buildkite/scripts/steps/build_api_docs.sh
+    label: 'Build API Docs'
+    agents:
+      queue: n2-4-spot
+    key: build_api_docs
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/code_coverage/daily.yml

@@ -13,7 +13,7 @@ steps:
       queue: kibana-default
     env:
       FTR_CONFIGS_DEPS: ''
-#      LIMIT_CONFIG_TYPE: 'unit,functional,integration'
+      #      LIMIT_CONFIG_TYPE: 'unit,functional,integration'
       LIMIT_CONFIG_TYPE: 'unit,integration'
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest.sh'
       JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/code_coverage/jest_integration.sh'
@@ -26,6 +26,6 @@ steps:
     depends_on:
       - jest
       - jest-integration
-#      - ftr-configs
+    #      - ftr-configs
     timeout_in_minutes: 30
     key: ingest

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -174,6 +174,12 @@ for (const testSuite of testSuites) {
         concurrency: concurrency,
         concurrency_group: process.env.UUID,
         concurrency_method: 'eager',
+        env: {
+          // disable split of test cases between parallel jobs when running them in flaky test runner
+          // by setting chunks vars to value 1, which means all test will run in one job
+          CLI_NUMBER: 1,
+          CLI_COUNT: 1,
+        },
       });
       break;
     default:

.buildkite/pipelines/on_merge.yml

@@ -11,9 +11,23 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
-
   - wait
 
+  - label: 'Triggering changes-based pipelines'
+    branches: main
+    agents:
+      queue: kibana-default
+    plugins:
+      - chronotc/monorepo-diff#v2.0.4:
+          watch:
+            - path:
+                - 'versions.json'
+              config:
+                command: 'node .buildkite/scripts/steps/trigger_pipeline.js kibana-buildkite-pipelines-deploy main'
+                label: 'Trigger pipeline deploy'
+                agents:
+                  queue: 'kibana-default'
+
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:

.buildkite/pipelines/performance/daily.yml

@@ -19,11 +19,17 @@ steps:
     depends_on: build
     key: tests
 
-  # - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
-  #   command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
-  #   agents:
-  #     queue: n2-2
-  #   depends_on: tests
+  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
+
+  - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats'
+    command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
 
   - wait: ~
     continue_on_failure: true

.buildkite/pull_requests.json

@@ -28,14 +28,14 @@
         "^\\.github/",
         "\\.md$",
         "\\.mdx$",
+        "^api_docs/.+\\.devdocs\\.json$",
         "^\\.backportrc\\.json$",
         "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
         "^\\.buildkite/pull_requests\\.json$"
       ],
       "always_require_ci_on_changed": [
-        "^docs/developer/plugin-list.asciidoc$",
-        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
+        "^docs/developer/plugin-list.asciidoc$"
       ],
       "kibana_versions_check": true
     }

.buildkite/scripts/build_kibana.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+source .buildkite/scripts/common/util.sh
+
 export KBN_NP_PLUGINS_BUILT=true
 
 echo "--- Build Kibana Distribution"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -87,6 +87,9 @@ EOF
 GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
 export GITHUB_TOKEN
 
+KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
+export KIBANA_CI_GITHUB_TOKEN
+
 KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
 export KIBANA_CI_REPORTER_KEY
 

.buildkite/scripts/steps/build_api_docs.sh

@@ -12,3 +12,24 @@ node scripts/build_ts_refs \
 
 echo "--- Build API Docs"
 node --max-old-space-size=12000 scripts/build_api_docs
+
+if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then
+  echo "--- Publish API Docs"
+
+  git config --global user.name kibanamachine
+  git config --global user.email '[email protected]'
+
+  branch="api_docs_$(date +%F_%H-%M-%S)"
+  git checkout -b "$branch"
+  git add ./*.docnav.json
+  git add api_docs
+  git commit -m "[api-docs] Daily api_docs build"
+
+  git remote add kibanamachine https://github.com/kibanamachine/kibana.git
+  git push -u kibanamachine "$branch"
+  prUrl=$(gh pr create --repo elastic/kibana --title "[api-docs] $(date +%F) Daily api_docs build" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "backport:auto-version")
+  echo "Opened PR: $prUrl"
+  gh pr merge --repo elastic/kibana --auto --squash "$prUrl"
+
+  GH_TOKEN="$KIBANA_CI_GITHUB_TOKEN" gh pr review --repo elastic/kibana --approve -b "Automated review from $BUILDKITE_BUILD_URL" "$prUrl"
+fi

.buildkite/scripts/steps/checks.sh

@@ -11,7 +11,6 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh
-.buildkite/scripts/steps/checks/doc_api_changes.sh
 .buildkite/scripts/steps/checks/kbn_pm_dist.sh
 .buildkite/scripts/steps/checks/plugin_list_docs.sh
 .buildkite/scripts/steps/checks/bundle_limits.sh

.buildkite/scripts/steps/code_coverage/ingest.sh

@@ -17,14 +17,16 @@ export HOST_FROM_VAULT
 TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
 export TIME_STAMP
 
-echo "--- Download previous git sha"
-.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
-PREVIOUS_SHA=$(cat downloaded_previous.txt)
+.buildkite/scripts/bootstrap.sh
 
-echo "--- Upload new git sha"
-.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh
+revolveBuildHashes() {
+  echo "--- Download previous git sha"
+  .buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
+  PREVIOUS_SHA=$(cat downloaded_previous.txt)
 
-.buildkite/scripts/bootstrap.sh
+  echo "--- Upload new git sha"
+  .buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh
+}
 
 collectRan() {
   buildkite-agent artifact download target/ran_files/* .
@@ -59,9 +61,9 @@ archiveReports() {
   local xs=("$@")
   for x in "${xs[@]}"; do
     echo "### Collect and Upload for: ${x}"
-#    fileHeads "target/file-heads-archive-reports-for-${x}.txt" "target/kibana-coverage/${x}"
-#    dirListing "target/dir-listing-${x}-combined-during-archiveReports.txt" target/kibana-coverage/${x}-combined
-#    dirListing "target/dir-listing-${x}-during-archiveReports.txt" target/kibana-coverage/${x}
+    #    fileHeads "target/file-heads-archive-reports-for-${x}.txt" "target/kibana-coverage/${x}"
+    #    dirListing "target/dir-listing-${x}-combined-during-archiveReports.txt" target/kibana-coverage/${x}-combined
+    #    dirListing "target/dir-listing-${x}-during-archiveReports.txt" target/kibana-coverage/${x}
     collectAndUpload "target/kibana-coverage/${x}/kibana-${x}-coverage.tar.gz" "target/kibana-coverage/${x}-combined"
   done
 }
@@ -86,25 +88,53 @@ mergeAll() {
   done
 }
 
+annotateForStaticSite() {
+  local xs=("$@")
+  local markdownLinks=()
+
+  OLDIFS="${IFS}"
+  IFS=$'\n'
+
+  for x in "${xs[@]}"; do
+    markdownLinks+=(" - [$x](https://kibana-coverage.elastic.dev/${TIME_STAMP}/${x}-combined/index.html)")
+  done
+
+  content=$(
+    cat <<-EOF
+### Browse the Code Coverage Static Site
+
+_Links are pinned to the current build number._
+
+${markdownLinks[*]}
+EOF
+  )
+
+  IFS="${OLDIFS}"
+
+  buildkite-agent annotate --style "info" --context 'ctx-coverage-static-site' "${content}"
+}
+
 modularize() {
   collectRan
   if [ -d target/ran_files ]; then
+    revolveBuildHashes
     uniqueifyRanConfigs "${ran[@]}"
     fetchArtifacts "${uniqRanConfigs[@]}"
     mergeAll "${uniqRanConfigs[@]}"
     archiveReports "${uniqRanConfigs[@]}"
     .buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh "${uniqRanConfigs[@]}"
     .buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh "${uniqRanConfigs[@]}"
+    annotateForStaticSite "${uniqRanConfigs[@]}"
     .buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh
     source .buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' \
       "${BUILDKITE_BUILD_NUMBER}" "${BUILDKITE_BUILD_URL}" "${PREVIOUS_SHA}" \
       'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
     ingestModular "${uniqRanConfigs[@]}"
+    annotateForKibanaLinks
   else
     echo "--- Found zero configs that ran, cancelling ingestion."
     exit 11
   fi
 }
 
 modularize
-echo "### unique ran configs: ${uniqRanConfigs[*]}"

.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh

@@ -29,6 +29,23 @@ echo "### debug TEAM_ASSIGN_PATH: ${TEAM_ASSIGN_PATH}"
 BUFFER_SIZE=500
 export BUFFER_SIZE
 
+annotateForKibanaLinks() {
+  local currentBuildNumber="$BUILDKITE_BUILD_NUMBER"
+  local coverageUrl="https://kibana-stats.elastic.dev/app/discover#/?_g=(filters:!(),query:(language:kuery,query:''),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))&_a=(columns:!(),filters:!(),hideChart:!f,index:'64419790-4218-11ea-b2d8-81bcbf78dfcb',interval:auto,query:(language:kuery,query:'BUILD_ID%20:%20${currentBuildNumber}'),sort:!(!('@timestamp',desc)))"
+  local totalCoverageUrl="https://kibana-stats.elastic.dev/app/discover#/?_g=(filters:!(),query:(language:kuery,query:''),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))&_a=(columns:!(),filters:!(),hideChart:!f,index:d78f9120-4218-11ea-b2d8-81bcbf78dfcb,interval:auto,query:(language:kuery,query:'BUILD_ID%20:%20${currentBuildNumber}'),sort:!(!('@timestamp',desc)))"
+
+  cat <<EOF | buildkite-agent annotate --style "info" --context 'ctx-kibana-links'
+### Browse the following url(s) to visually verify in Kibana
+
+_Links are pinned to the current build number._
+
+  - [Code Coverage]($coverageUrl)
+  - [Total Code Coverage]($totalCoverageUrl)
+
+EOF
+
+}
+
 ingestModular() {
   local xs=("$@")
 

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -96,7 +96,7 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config');
       cd "${destination}"
       gsutil -m cp -r *.* gs://${BASE_BUCKET_DAILY}/${DESTINATION}
       cp manifest.json manifest-latest.json
-      gsutil cp manifest-latest.json gs://${BASE_BUCKET_DAILY}/${VERSION}
+      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp manifest-latest.json gs://${BASE_BUCKET_DAILY}/${VERSION}
 
       buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST 'https://storage.googleapis.com/${BASE_BUCKET_DAILY}/${DESTINATION}/manifest.json'
       buildkite-agent meta-data set ES_SNAPSHOT_VERSION '${VERSION}'

.buildkite/scripts/steps/es_snapshots/promote.sh

@@ -15,5 +15,5 @@ node "$(dirname "${0}")/promote_manifest.js" "$ES_SNAPSHOT_MANIFEST"
 
 if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
   echo "--- Trigger agent packer cache pipeline"
-  node .buildkite/scripts/steps/trigger_packer_cache.js
+  node .buildkite/scripts/steps/trigger_pipeline.js kibana-agent-packer-cache main
 fi

.buildkite/scripts/steps/es_snapshots/promote_manifest.js

@@ -39,11 +39,11 @@ const { BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT } = require('./bucket_config');
       `
       set -euo pipefail
       cp manifest.json manifest-latest-verified.json
-      gsutil cp manifest-latest-verified.json gs://${BASE_BUCKET_DAILY}/${version}/
+      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp manifest-latest-verified.json gs://${BASE_BUCKET_DAILY}/${version}/
       rm manifest.json
       cp manifest-permanent.json manifest.json
       gsutil -m cp -r gs://${bucket}/* gs://${BASE_BUCKET_PERMANENT}/${version}/
-      gsutil cp manifest.json gs://${BASE_BUCKET_PERMANENT}/${version}/
+      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp manifest.json gs://${BASE_BUCKET_PERMANENT}/${version}/
     `,
       { shell: '/bin/bash' }
     );

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -15,6 +15,21 @@ node scripts/es snapshot&
 
 esPid=$!
 
+# unset env vars defined in other parts of CI for automatic APM collection of
+# Kibana. We manage APM config in our FTR config and performance service, and
+# APM treats config in the ENV with a very high precedence.
+unset ELASTIC_APM_ENVIRONMENT
+unset ELASTIC_APM_TRANSACTION_SAMPLE_RATE
+unset ELASTIC_APM_SERVER_URL
+unset ELASTIC_APM_SECRET_TOKEN
+unset ELASTIC_APM_ACTIVE
+unset ELASTIC_APM_CONTEXT_PROPAGATION_ONLY
+unset ELASTIC_APM_ACTIVE
+unset ELASTIC_APM_SERVER_URL
+unset ELASTIC_APM_SECRET_TOKEN
+unset ELASTIC_APM_GLOBAL_LABELS
+
+
 export TEST_ES_URL=http://elastic:changeme@localhost:9200
 export TEST_ES_DISABLE_STARTUP=true
 
@@ -30,19 +45,19 @@ for i in "${journeys[@]}"; do
 
     checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \
       node scripts/functional_tests \
-      --config "x-pack/test/performance/journeys/${i}/config.ts" \
-      --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-      --debug \
-      --bail
+        --config "x-pack/test/performance/journeys/${i}/config.ts" \
+        --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+        --debug \
+        --bail
 
     export TEST_PERFORMANCE_PHASE=TEST
 
     checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: TEST)" \
       node scripts/functional_tests \
-      --config "x-pack/test/performance/journeys/${i}/config.ts" \
-      --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-      --debug \
-      --bail
+        --config "x-pack/test/performance/journeys/${i}/config.ts" \
+        --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+        --debug \
+        --bail
 done
 
 kill "$esPid"

.buildkite/scripts/steps/functional/report_performance_metrics.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+# TODO: Add new user and change lines accordingly
+USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/ci_stats_performance_metrics)"
+PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/ci_stats_performance_metrics)"
+APM_SERVER_URL="https://kibana-ops-e2e-perf.kb.us-central1.gcp.cloud.es.io:9243/internal/apm"
+BUILD_ID=${BUILDKITE_BUILD_ID}
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Extract APM metrics & report them to ci-stats"
+
+node scripts/report_performance_metrics \
+    --buildId "${BUILD_ID}" \
+    --apm-url "${APM_SERVER_URL}" \
+    --apm-username "${USER_FROM_VAULT}" \
+    --apm-password "${PASS_FROM_VAULT}"