Merge branch 'main' into migrateScrollCount

.buildkite/scripts/steps/checks.sh

@@ -11,7 +11,6 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh
-.buildkite/scripts/steps/checks/kbn_pm_dist.sh
 .buildkite/scripts/steps/checks/plugin_list_docs.sh
 .buildkite/scripts/steps/checks/bundle_limits.sh
 .buildkite/scripts/steps/checks/i18n.sh

.buildkite/scripts/steps/checks/test_projects.sh

@@ -6,4 +6,4 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Test Projects
 checks-reporter-with-killswitch "Test Projects" \
-  yarn kbn run test --exclude kibana --oss --skip-kibana-plugins --skip-missing
+  yarn kbn run-in-packages test

.github/CODEOWNERS

@@ -372,6 +372,7 @@
 /x-pack/test/cases_api_integration/ @elastic/response-ops
 /x-pack/test/functional/services/cases/ @elastic/response-ops
 /x-pack/test/functional_with_es_ssl/apps/cases/ @elastic/response-ops
+/x-pack/test/api_integration/apis/cases @elastic/response-ops
 
 # Enterprise Search
 /x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend

docs/developer/getting-started/monorepo-packages.asciidoc

@@ -17,24 +17,14 @@ Remember that any time you need to make sure the monorepo is ready to be used ju
 yarn kbn bootstrap
 ----
 
-[discrete]
-=== Building Non Bazel Packages
-
-Non Bazel packages can be built independently with
-
-[source,bash]
-----
-yarn kbn run build -i PACKAGE_NAME
-----
-
 [discrete]
 === Building Bazel Packages
 
 Bazel packages are built as a whole for now. You can use:
 
 [source,bash]
 ----
-yarn kbn build
+yarn kbn bootstrap
 ----
 
 [discrete]

kbn_pm/README.mdx

@@ -0,0 +1,44 @@
+---
+id: kibDevDocsOpsKbnPm
+slug: /kibana-dev-docs/ops/kbn-pm
+title: "@kbn/pm"
+description: 'The tool which bootstraps the repo and helps work with packages'
+date: 2022-07-14
+tags: ['kibana', 'dev', 'contributor', 'operations', 'packages', 'scripts']
+---
+
+`@kbn/pm` is the tool that we use to bootstrap the Kibana repository, build packages with Bazel, and run scripts in packages.
+
+## commands
+
+### `yarn kbn bootstrap`
+
+Use this command to install dependencies, build packages, and prepare the repo for local development.
+
+### `yarn kbn watch`
+
+Use this command to build all packages and make sure that they are rebuilt as you make changes.
+
+### and more!
+
+There are several commands supported by `@kbn/pm`, but rather than documenting them here they are documented in the help text. Please run `yarn kbn --help` locally to see the most up-to-date info.
+
+## Why isn't this TypeScript?
+
+Since this tool is required for bootstrapping the repository it needs to work without any dependencies installed and without a build toolchain. We accomplish this by writing the tool in vanilla JS (shocker!) and using TypeScript to validate the code which is typed via heavy use of JSDoc comments.
+
+In order to use import/export syntax and enhance the developer experience a little we use the `.mjs` file extension.
+
+In some cases we actually do use TypeScript files, just for defining complicated types. These files are then imported only in special TS-compatible JSDoc comments, so Node.js will never try to import them but they can be used to define types which are too complicated to define inline or in a JSDoc comment.
+
+There are cases where `@kbn/pm` relies on code from packages, mostly to prevent reimplementing common functionality. This can only be done in one of two ways:
+
+ 1. With a dynamic `await import(...)` statement that is always run after boostrap is complete, or is wrapped in a try/catch in case bootstrap didn't complete successfully.
+ 2. By pulling in the source code of the un-built package.
+
+Option 1 is used in several places, with contingencies in place in case bootstrap failed. Option 2 is used for two pieces of code which are needed in order to run bootstrap:
+
+ 1. `@kbn/plugin-discovery` as we need to populate the `@kbn/synthetic-package-map` to run Bazel
+ 2. `@kbn/bazel-runner` as we want to have the logic for running bazel in a single location
+
+Because we load these two packages from source, without being built, before bootstrap is ever run, they can not depend on other packages and must be written in Vanilla JS as well.

kbn_pm/src/cli.mjs

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * This is the script that's run by `yarn kbn`. This script has as little logic
+ * as possible so that it can:
+ *  - run without being built and without any dependencies
+ *  - can bootstrap the repository, installing all deps and building all packages
+ *  - load additional commands from packages which will extend the functionality
+ *    beyond bootstrapping
+ */
+
+import { Args } from './lib/args.mjs';
+import { getHelp } from './lib/help.mjs';
+import { createFlagError, isCliError } from './lib/cli_error.mjs';
+import { COMMANDS } from './commands/index.mjs';
+import { Log } from './lib/log.mjs';
+
+const start = Date.now();
+const args = new Args(process.argv.slice(2), process.env.CI ? ['--quiet'] : []);
+const log = new Log(args.getLoggingLevel());
+const cmdName = args.getCommandName();
+
+/**
+ * @param {import('./lib/log.mjs').Log} log
+ */
+async function tryToGetCiStatsReporter(log) {
+  try {
+    const { CiStatsReporter } = await import('@kbn/ci-stats-reporter');
+    return CiStatsReporter.fromEnv(log);
+  } catch {
+    return;
+  }
+}
+
+try {
+  const cmd = cmdName ? COMMANDS.find((c) => c.name === cmdName) : undefined;
+
+  if (cmdName && !cmd) {
+    throw createFlagError(`Invalid command name [${cmdName}]`);
+  }
+
+  if (args.getBooleanValue('help')) {
+    log._write(await getHelp(cmdName));
+    process.exit(0);
+  }
+
+  if (!cmd) {
+    throw createFlagError('missing command name');
+  }
+
+  /** @type {import('@kbn/ci-stats-reporter').CiStatsTiming[]} */
+  const timings = [];
+
+  /** @type {import('./lib/command').CommandRunContext['time']} */
+  const time = async (id, block) => {
+    if (!cmd.reportTimings) {
+      return await block();
+    }
+
+    const start = Date.now();
+    log.verbose(`[${id}]`, 'start');
+    const [result] = await Promise.allSettled([block()]);
+    const ms = Date.now() - start;
+    log.verbose(`[${id}]`, result.status === 'fulfilled' ? 'success' : 'failure', 'in', ms, 'ms');
+    timings.push({
+      group: cmd.reportTimings.group,
+      id,
+      ms,
+      meta: {
+        success: result.status === 'fulfilled',
+      },
+    });
+
+    if (result.status === 'fulfilled') {
+      return result.value;
+    } else {
+      throw result.reason;
+    }
+  };
+
+  const [result] = await Promise.allSettled([
+    (async () =>
+      await cmd.run({
+        args,
+        log,
+        time,
+      }))(),
+  ]);
+
+  if (cmd.reportTimings) {
+    timings.push({
+      group: cmd.reportTimings.group,
+      id: cmd.reportTimings.id,
+      ms: Date.now() - start,
+      meta: {
+        success: result.status === 'fulfilled',
+      },
+    });
+  }
+
+  if (timings.length) {
+    const reporter = await tryToGetCiStatsReporter(log);
+    if (reporter) {
+      await reporter.timings({ timings });
+    }
+  }
+
+  if (result.status === 'rejected') {
+    throw result.reason;
+  }
+} catch (error) {
+  if (!isCliError(error)) {
+    throw error;
+  }
+
+  log.error(`[${cmdName}] failed: ${error.message}`);
+
+  if (error.showHelp) {
+    log._write('');
+    log._write(await getHelp(cmdName));
+  }
+
+  process.exit(error.exitCode ?? 1);
+}

kbn_pm/src/commands/bootstrap/bootstrap_command.mjs

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { spawnSync } from '../../lib/spawn.mjs';
+import * as Bazel from '../../lib/bazel.mjs';
+import { haveNodeModulesBeenManuallyDeleted, removeYarnIntegrityFileIfExists } from './yarn.mjs';
+import { setupRemoteCache } from './setup_remote_cache.mjs';
+import { regenerateSyntheticPackageMap } from './regenerate_synthetic_package_map.mjs';
+import { sortPackageJson } from './sort_package_json.mjs';
+import { pluginDiscovery } from './plugins.mjs';
+import { regenerateBaseTsconfig } from './regenerate_base_tsconfig.mjs';
+
+/** @type {import('../../lib/command').Command} */
+export const command = {
+  name: 'bootstrap',
+  intro: 'Bootstrap the Kibana repository, installs all dependencies and builds all packages',
+  description: `
+    This command should be run every time you checkout a new revision, or can be used to build all packages
+    once after making a change locally. Package builds are cached remotely so when you don't have local
+    changes build artifacts will be downloaded from the remote cache.
+  `,
+  flagsHelp: `
+    --force-install      Use this flag to force bootstrap to install yarn dependencies. By default the',
+                          command will attempt to only run yarn installs when necessary, but if you manually',
+                          delete the node modules directory or have an issue in your node_modules directory',
+                          you might need to force the install manually.',
+    --offline            Run the installation process without consulting online resources. This is useful and',
+                          sometimes necessary for using bootstrap on an airplane for instance. The local caches',
+                          will be used exclusively, including a yarn-registry local mirror which is created and',
+                          maintained by successful online bootstrap executions.',
+    --no-validate        By default bootstrap validates the yarn.lock file to check for a handfull of',
+                          conditions. If you run into issues with this process locally you can disable it by',
+                          passing this flag.
+    --no-vscode          By default bootstrap updates the .vscode directory to include commonly useful vscode
+                          settings for local development. Disable this process either pass this flag or set
+                          the KBN_BOOTSTRAP_NO_VSCODE=true environment variable.
+    --quiet              Prevent logging more than basic success/error messages
+  `,
+  reportTimings: {
+    group: 'scripts/kbn bootstrap',
+    id: 'total',
+  },
+  async run({ args, log, time }) {
+    const offline = args.getBooleanValue('offline') ?? false;
+    const validate = args.getBooleanValue('validate') ?? true;
+    const quiet = args.getBooleanValue('quiet') ?? false;
+    const vscodeConfig =
+      args.getBooleanValue('vscode') ?? (process.env.KBN_BOOTSTRAP_NO_VSCODE ? false : true);
+
+    // Force install is set in case a flag is passed into yarn kbn bootstrap or
+    // our custom logic have determined there is a chance node_modules have been manually deleted and as such bazel
+    // tracking mechanism is no longer valid
+    const forceInstall =
+      args.getBooleanValue('force-install') ?? haveNodeModulesBeenManuallyDeleted();
+
+    Bazel.tryRemovingBazeliskFromYarnGlobal(log);
+
+    // Install bazel machinery tools if needed
+    Bazel.ensureInstalled(log);
+
+    // Setup remote cache settings in .bazelrc.cache if needed
+    setupRemoteCache(log);
+
+    // Bootstrap process for Bazel packages
+    // Bazel is now managing dependencies so yarn install
+    // will happen as part of this
+    //
+    // NOTE: Bazel projects will be introduced incrementally
+    // And should begin from the ones with none dependencies forward.
+    // That way non bazel projects could depend on bazel projects but not the other way around
+    // That is only intended during the migration process while non Bazel projects are not removed at all.
+    if (forceInstall) {
+      await time('force install dependencies', async () => {
+        removeYarnIntegrityFileIfExists();
+        await Bazel.expungeCache(log, { quiet });
+        await Bazel.installYarnDeps(log, { offline, quiet });
+      });
+    }
+
+    const plugins = await time('plugin discovery', async () => {
+      return pluginDiscovery();
+    });
+
+    // generate the synthetic package map which powers several other features, needed
+    // as an input to the package build
+    await time('regenerate synthetic package map', async () => {
+      regenerateSyntheticPackageMap(plugins);
+    });
+
+    // build packages
+    await time('build packages', async () => {
+      await Bazel.buildPackages(log, { offline, quiet });
+    });
+
+    await time('sort package json', async () => {
+      await sortPackageJson();
+    });
+    await time('regenerate tsconfig.base.json', async () => {
+      regenerateBaseTsconfig(plugins);
+    });
+
+    if (validate) {
+      // now that packages are built we can import `@kbn/yarn-lock-validator`
+      const { readYarnLock, validateDependencies } = await import('@kbn/yarn-lock-validator');
+      const yarnLock = await time('read yarn.lock', async () => {
+        return await readYarnLock();
+      });
+      await time('validate dependencies', async () => {
+        await validateDependencies(log, yarnLock);
+      });
+    }
+
+    if (vscodeConfig) {
+      await time('update vscode config', async () => {
+        // Update vscode settings
+        spawnSync('node', ['scripts/update_vscode_config']);
+
+        log.success('vscode config updated');
+      });
+    }
+  },
+};