Merge remote-tracking branch 'upstream/master' into ci-stats-fixes

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -0,0 +1,52 @@
+const stepInput = (key, nameOfSuite) => {
+  return {
+    key: `ftsr-suite/${key}`,
+    text: nameOfSuite,
+    required: false,
+    default: '0',
+  };
+};
+
+const OSS_CI_GROUPS = 12;
+const XPACK_CI_GROUPS = 13;
+
+const inputs = [
+  {
+    key: 'ftsr-override-count',
+    text: 'Override for all suites',
+    default: 0,
+    required: true,
+  },
+  {
+    key: 'ftsr-concurrency',
+    text: 'Max concurrency per step',
+    default: 20,
+    required: true,
+  },
+];
+
+for (let i = 1; i <= OSS_CI_GROUPS; i++) {
+  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+}
+
+for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
+  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
+}
+
+const pipeline = {
+  steps: [
+    {
+      input: 'Number of Runs',
+      fields: inputs,
+    },
+    {
+      wait: '~',
+    },
+    {
+      command: '.buildkite/pipelines/flaky_tests/runner.sh',
+      label: 'Create pipeline',
+    },
+  ],
+};
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+node .buildkite/pipelines/flaky_tests/pipeline.js | buildkite-agent pipeline upload

.buildkite/pipelines/flaky_tests/runner.js

@@ -0,0 +1,82 @@
+const { execSync } = require('child_process');
+
+const keys = execSync('buildkite-agent meta-data keys')
+  .toString()
+  .split('\n')
+  .filter((k) => k.startsWith('ftsr-suite/'));
+
+const overrideCount = parseInt(
+  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
+);
+
+const concurrency =
+  parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20;
+
+const testSuites = [];
+for (const key of keys) {
+  if (!key) {
+    continue;
+  }
+
+  const value =
+    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+  testSuites.push({
+    key: key.replace('ftsr-suite/', ''),
+    count: value === '' ? defaultCount : parseInt(value),
+  });
+}
+
+const steps = [];
+const pipeline = {
+  env: {
+    IGNORE_SHIP_CI_STATS_ERROR: 'true',
+  },
+  steps: steps,
+};
+
+steps.push({
+  command: '.buildkite/scripts/steps/build_kibana.sh',
+  label: 'Build Kibana Distribution and Plugins',
+  agents: { queue: 'c2-8' },
+  key: 'build',
+  if: "build.env('BUILD_ID_FOR_ARTIFACTS') == null || build.env('BUILD_ID_FOR_ARTIFACTS') == ''",
+});
+
+for (const testSuite of testSuites) {
+  const TEST_SUITE = testSuite.key;
+  const RUN_COUNT = testSuite.count;
+  const UUID = TEST_SUITE + process.env.UUID;
+
+  const JOB_PARTS = TEST_SUITE.split('/');
+  const IS_XPACK = JOB_PARTS[0] === 'xpack';
+  const CI_GROUP = JOB_PARTS.length > 2 ? JOB_PARTS[2] : '';
+
+  if (RUN_COUNT < 1) {
+    continue;
+  }
+
+  if (IS_XPACK) {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
+      label: `Default CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-6' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  } else {
+    steps.push({
+      command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
+      label: `OSS CI Group ${CI_GROUP}`,
+      agents: { queue: 'ci-group-4d' },
+      depends_on: 'build',
+      parallelism: RUN_COUNT,
+      concurrency: concurrency,
+      concurrency_group: UUID,
+    });
+  }
+}
+
+console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/runner.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+UUID="$(cat /proc/sys/kernel/random/uuid)"
+export UUID
+
+node .buildkite/pipelines/flaky_tests/runner.js | buildkite-agent pipeline upload

dev_docs/contributing/standards.mdx

@@ -0,0 +1,74 @@
+---
+id: kibStandards
+slug: /kibana-dev-docs/standards
+title: Standards and guidelines
+summary: Standards and guidelines we expect every Kibana developer to abide by
+date: 2021-09-28
+tags: ['contributor', 'dev', 'github', 'getting started', 'onboarding', 'kibana']
+---
+
+## Developer principles
+
+We expect all developers to read and abide by our overarching <DocLink id="kibDeveloperPrinciples" />.
+
+## Style guide
+
+Please read and abide by our <DocLink id="kibStyleGuide" text="Style guide" />. The majority of these items are linted against but some are not.
+
+## RESTful HTTP APIs
+
+### Terminology
+
+**REST APIs**
+Technically, REST does not specify a protocol, but for readability, we’ll be calling RESTful HTTP APIs as REST APIs for short for the remainder of the section. HTTP APIs that serve HTML, CSS and images are not REST APIs.
+
+**End user**
+Anywhere we refer to “end user” in this section, we are referring to someone who is using the REST APIs. The distinction between Product breaking changes and plugin breaking changes can also be found in this [Make it Minor strawman proposal doc](https://docs.google.com/document/d/12R0w75dSNR-VDQLGl2vxFyEHhzxNT38iamYhven9uvw/edit). This can be a tricky distinction, as some folks may consider end user to only be folks that use the Kibana UI.
+
+### Privacy
+
+| Type     | Description                                                                                                                                          | Guarantees                                                                      |
+| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
+| Internal | An API with “internal” in the route. Specifically it should be `/internal/{pluginname}/{...}`. It should only be used by the plugin that created it. | None                                                                            |
+| Public   | Any API that is not internal based on above definition                                                                                               | Based on <DocLink id="kibStandards" section="release-tags" text="release tag"/> |
+
+### Do not access directly from plugins
+
+Plugins should not attempt to directly access the REST APIs created by another plugin. The plugin author who registered the public REST API should provide access to it via functionality on the plugin lifecycle contract. Accessing functionality through a client side plugin contract provides more type-safety compared to calling the REST API directly. Never attempt to call a server-side REST API if you are already on the server-side. It will not work. This should also be avoided in any code provided within a common folder.
+
+### Path names
+
+All public API path names should start with `/api/`.
+All internal APIs should start with `/internal/{pluginname}/{...}`.
+
+### Backward compatibility and breaking changes
+
+Every public API should have a release tag specified at the top of it’s documentation page. Release tags are not applicable to internal APIs, as we make no guarantees on those.
+
+#### Release tags
+
+| Type | Description | Documentation | Asciidoc Tag |
+| Undocumented | Every public API should be documented, but if it isn’t, we make no guarantees about it. These need to be eliminated and should become internal or documented. |
+| Experimental | A public API that may break or be removed at any time. | experimental[] |
+| Beta | A public API that we make a best effort not to break or remove. However, there are no guarantees. | beta[] |
+| Stable | No breaking changes outside of a Major\* | stable[] |
+| Deprecated | Do not use, will be removed. | deprecated[] |
+
+\*This is likely to change with Make it Minor as we move towards a calendar based rolling deprecation and removal policy.
+
+#### What constitutes a breaking change?
+
+- A path change
+- A request payload change that adds a new required variable, or changes an existing one (new optional parameters are not breaking).
+- A response payload change that removes data or changes existing data (returning additional data is not breaking).
+- Status code changes
+
+### Telemetry
+
+Every team should be collecting telemetry metrics on it’s public API usage. This will be important for knowing when it’s safe to make breaking changes. The Core team will be looking into ways to make this easier and an automatic part of registration (see [#112291](https://github.com/elastic/kibana/issues/112291)).
+
+### Documentation
+
+Every public API should be documented inside the [docs/api](https://github.com/elastic/kibana/tree/master/docs/api) folder in asciidoc (this content will eventually be migrated to mdx to support the new docs system). If a public REST API is undocumented, you should either document it, or make it internal.
+
+Every public API should have a release tag specified using the appropriate documentation release tag above. If you do this, the docs system will provide a pop up explaining the conditions. If an API is not marked, it should be considered experimental.

docs/setup/install/deb.asciidoc

@@ -157,7 +157,6 @@ locations for a Debian-based system:
   | Configuration files including `kibana.yml`
   | /etc/kibana
   | <<settings,KBN_PATH_CONF>>
- d|
 
 | data
   | The location of the data files written to disk by Kibana and its plugins

package.json

@@ -771,6 +771,7 @@
     "multimatch": "^4.0.0",
     "mutation-observer": "^1.0.3",
     "ncp": "^2.0.0",
+    "node-jq": "2.0.0",
     "node-sass": "^6.0.1",
     "null-loader": "^3.0.0",
     "nyc": "^15.0.1",

packages/kbn-logging/src/ecs/agent.ts

@@ -7,7 +7,7 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-agent.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-agent.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/autonomous_system.ts

@@ -7,7 +7,7 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-as.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-as.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/base.ts

@@ -7,7 +7,7 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-base.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-base.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/client.ts

@@ -17,7 +17,7 @@ interface NestedFields {
 }
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-client.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-client.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/cloud.ts

@@ -7,7 +7,7 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-cloud.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-cloud.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/code_signature.ts

@@ -7,15 +7,17 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-code_signature.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-code_signature.html
  *
  * @internal
  */
 export interface EcsCodeSignature {
+  digest_algorithm?: string;
   exists?: boolean;
   signing_id?: string;
   status?: string;
   subject_name?: string;
+  timestamp?: string;
   team_id?: string;
   trusted?: boolean;
   valid?: boolean;

packages/kbn-logging/src/ecs/container.ts

@@ -7,14 +7,21 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-container.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-container.html
  *
  * @internal
  */
 export interface EcsContainer {
+  cpu?: { usage?: number };
+  disk?: Disk;
   id?: string;
   image?: { name?: string; tag?: string[] };
   labels?: Record<string, unknown>;
   name?: string;
   runtime?: string;
 }
+
+interface Disk {
+  read?: { bytes?: number };
+  write?: { bytes?: number };
+}

packages/kbn-logging/src/ecs/data_stream.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-data_stream.html
+ *
+ * @internal
+ */
+export interface EcsDataStream {
+  dataset?: string;
+  namespace?: string;
+  type?: 'logs' | 'metrics';
+}

packages/kbn-logging/src/ecs/destination.ts

@@ -17,7 +17,7 @@ interface NestedFields {
 }
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-destination.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-destination.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/dll.ts

@@ -17,7 +17,7 @@ interface NestedFields {
 }
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-dll.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-dll.html
  *
  * @internal
  */

packages/kbn-logging/src/ecs/dns.ts

@@ -7,7 +7,7 @@
  */
 
 /**
- * https://www.elastic.co/guide/en/ecs/1.9/ecs-dns.html
+ * https://www.elastic.co/guide/en/ecs/1.12/ecs-dns.html
  *
  * @internal
  */