Merge branch 'master' into ml-disable-scripted-fields

.ci/jobs.yml

@@ -1,4 +1,4 @@
-# This file is needed by functionalTests:ensureAllTestsInCiGroup for the list of ciGroups. That must be changed before this file can be removed
+# This file is needed by node scripts/ensure_all_tests_in_ci_group  for the list of ciGroups. That must be changed before this file can be removed
 
 JOB:
   - kibana-intake

docs/user/alerting/alerting-getting-started.asciidoc

@@ -123,14 +123,15 @@ image::images/alert-concepts-connectors.svg[Connectors provide a central place t
 [float]
 === Summary
 
-An _alert_ consists of conditions, _actions_, and a schedule. When conditions are met, _alert instances_ are created that render _actions_ and invoke them. To make action setup and update easier, actions refer to _connectors_ that centralize the information used to connect with {kib} services and third-party integrations.
+An _alert_ consists of conditions, _actions_, and a schedule. When conditions are met, _alert instances_ are created that render _actions_ and invoke them. To make action setup and update easier, actions refer to _connectors_ that centralize the information used to connect with {kib} services and third-party integrations. The following example ties these concepts together: 
 
 image::images/alert-concepts-summary.svg[Alerts, actions, alert instances and connectors work together to convert detection into action]
 
-* *Alert*: a specification of the conditions to be detected, the schedule for detection, and the response when detection occurs. 
-* *Action*: the response to a detected condition defined in the alert. Typically actions specify a service or third party integration along with alert details that will be sent to it. 
-* *Alert instance*: state tracked by {kib} for every occurrence of a detected condition. Actions as well as controls like muting and re-notification are controlled at the instance level. 
-* *Connector*: centralized configurations for services and third party integration that are referenced by actions. 
+. Anytime an *alert*'s conditions are met, an *alert instance* is created.  This example checks for servers with average CPU > 0.9. Three servers meet the condition, so three instances are created. 
+. Instances create *actions* as long as they are not muted or throttled. When actions are created, the template that was setup in the alert is filled with actual values.  In this example three actions are created, and the template string {{server}} is replaced with the server name for each instance. 
+. {kib} invokes the actions, sending them to a 3rd party *integration* like an email service.
+. If the 3rd party integration has connection parameters or credentials, {kib} will fetch these from the *connector* referenced in the action. 
+
 
 [float]
 [[alerting-concepts-differences]]

package.json

@@ -100,7 +100,7 @@
     "@babel/core": "^7.11.6",
     "@babel/runtime": "^7.11.2",
     "@elastic/datemath": "link:packages/elastic-datemath",
-    "@elastic/elasticsearch": "7.10.0",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary",
     "@elastic/ems-client": "7.11.0",
     "@elastic/eui": "30.6.0",
     "@elastic/filesaver": "1.1.2",
@@ -824,7 +824,7 @@
     "url-loader": "^2.2.0",
     "use-resize-observer": "^6.0.0",
     "val-loader": "^1.1.1",
-    "vega": "^5.17.0",
+    "vega": "^5.17.1",
     "vega-lite": "^4.17.0",
     "vega-schema-url-parser": "^2.1.0",
     "vega-tooltip": "^0.24.2",

scripts/ensure_all_tests_in_ci_group.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_ensure_all_tests_in_ci_group');

src/core/public/doc_links/doc_links_service.ts

@@ -147,6 +147,7 @@ export class DocLinksService {
           featureImportance: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-feature-importance.html`,
           outlierDetectionRoc: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfanalytics-evaluate.html#ml-dfanalytics-roc`,
           regressionEvaluation: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfanalytics-evaluate.html#ml-dfanalytics-regression-evaluation`,
+          classificationAucRoc: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfanalytics-evaluate.html#ml-dfanalytics-class-aucroc`,
         },
         transforms: {
           guide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/reference/${DOC_LINK_VERSION}/transforms.html`,

tasks/function_test_groups.js → src/dev/run_ensure_all_tests_in_ci_group.js

@@ -21,32 +21,28 @@ import { readFileSync } from 'fs';
 import { resolve } from 'path';
 
 import execa from 'execa';
-import grunt from 'grunt';
 import { safeLoad } from 'js-yaml';
 
-const JOBS_YAML = readFileSync(resolve(__dirname, '../.ci/jobs.yml'), 'utf8');
+import { run } from '@kbn/dev-utils';
+
+const JOBS_YAML = readFileSync(resolve(__dirname, '../../.ci/jobs.yml'), 'utf8');
 const TEST_TAGS = safeLoad(JOBS_YAML)
   .JOB.filter((id) => id.startsWith('kibana-ciGroup'))
   .map((id) => id.replace(/^kibana-/, ''));
 
-grunt.registerTask(
-  'functionalTests:ensureAllTestsInCiGroup',
-  'Check that all of the functional tests are in a CI group',
-  async function () {
-    const done = this.async();
-
-    try {
-      const result = await execa(process.execPath, [
-        'scripts/functional_test_runner',
-        ...TEST_TAGS.map((tag) => `--include-tag=${tag}`),
-        '--config',
-        'test/functional/config.js',
-        '--test-stats',
-      ]);
-      const stats = JSON.parse(result.stderr);
-
-      if (stats.excludedTests.length > 0) {
-        grunt.fail.fatal(`
+run(async ({ log }) => {
+  try {
+    const result = await execa(process.execPath, [
+      'scripts/functional_test_runner',
+      ...TEST_TAGS.map((tag) => `--include-tag=${tag}`),
+      '--config',
+      'test/functional/config.js',
+      '--test-stats',
+    ]);
+    const stats = JSON.parse(result.stderr);
+
+    if (stats.excludedTests.length > 0) {
+      log.error(`
           ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that
           all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js"
           knows about the tag that you are using.
@@ -55,12 +51,11 @@ grunt.registerTask(
 
           - ${stats.excludedTests.join('\n          - ')}
         `);
-        return;
-      }
-
-      done();
-    } catch (error) {
-      grunt.fail.fatal(error.stack);
+      process.exitCode = 1;
+      return;
     }
+  } catch (error) {
+    log.error(error.stack);
+    process.exitCode = 1;
   }
-);
+});

src/dev/run_find_plugins_with_circular_deps.ts

@@ -32,9 +32,7 @@ type CircularDepList = Set<string>;
 
 const allowedList: CircularDepList = new Set([
   'src/plugins/charts -> src/plugins/discover',
-  'src/plugins/charts -> src/plugins/vis_default_editor',
   'src/plugins/vis_default_editor -> src/plugins/visualizations',
-  'src/plugins/vis_default_editor -> src/plugins/visualize',
   'src/plugins/visualizations -> src/plugins/visualize',
   'x-pack/plugins/actions -> x-pack/plugins/case',
   'x-pack/plugins/case -> x-pack/plugins/security_solution',

src/plugins/charts/kibana.json

@@ -3,6 +3,5 @@
   "version": "kibana",
   "server": true,
   "ui": true,
-  "requiredPlugins": ["expressions"],
-  "requiredBundles": ["visDefaultEditor"]
+  "requiredPlugins": ["expressions"]
 }

src/plugins/charts/public/static/components/index.ts

@@ -17,17 +17,8 @@
  * under the License.
  */
 
-export { BasicOptions } from './basic_options';
 export { ColorMode, LabelRotation, defaultCountLabel } from './collections';
-export { ColorRanges, SetColorRangeValue } from './color_ranges';
-export { ColorSchemaOptions, SetColorSchemaOptionsValue } from './color_schema';
 export { ColorSchemaParams, Labels, Style } from './types';
-export { NumberInputOption } from './number_input';
-export { RangeOption } from './range';
-export { RequiredNumberInputOption } from './required_number_input';
-export { SelectOption } from './select';
-export { SwitchOption } from './switch';
-export { TextInputOption } from './text_input';
 export { LegendToggle } from './legend_toggle';
 export { ColorPicker } from './color_picker';
 export { CurrentTime } from './current_time';

src/plugins/data/common/index_patterns/index_pattern.stub.ts

@@ -25,6 +25,7 @@ export const stubIndexPattern: IIndexPattern = {
   fields: stubFields,
   title: 'logstash-*',
   timeFieldName: '@timestamp',
+  getTimeField: () => ({ name: '@timestamp', type: 'date' }),
 };
 
 export const stubIndexPatternWithFields: IIndexPattern = {

src/plugins/data/common/search/aggs/buckets/date_histogram.ts

@@ -149,7 +149,7 @@ export const getDateHistogramBucketAgg = ({
         type: 'field',
         filterFieldTypes: KBN_FIELD_TYPES.DATE,
         default(agg: IBucketDateHistogramAggConfig) {
-          return agg.getIndexPattern().timeFieldName;
+          return agg.getIndexPattern().getTimeField?.()?.name;
         },
         onChange(agg: IBucketDateHistogramAggConfig) {
           if (isAutoInterval(get(agg, 'params.interval')) && !agg.fieldIsTimeField()) {

src/plugins/data/server/kql_telemetry/usage_collector/fetch.test.ts

@@ -18,7 +18,7 @@
  */
 
 import { fetchProvider } from './fetch';
-import { LegacyAPICaller } from 'kibana/server';
+import { ElasticsearchClient } from 'kibana/server';
 import { CollectorFetchContext } from 'src/plugins/usage_collection/server';
 import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks';
 
@@ -30,42 +30,41 @@ jest.mock('../../../common', () => ({
 }));
 
 let fetch: ReturnType<typeof fetchProvider>;
-let callCluster: LegacyAPICaller;
+let esClient: ElasticsearchClient;
 let collectorFetchContext: CollectorFetchContext;
 const collectorFetchContextMock = createCollectorFetchContextMock();
 
 function setupMockCallCluster(
   optCount: { optInCount?: number; optOutCount?: number } | null,
   language: string | undefined | null
 ) {
-  callCluster = (jest.fn((method, params) => {
-    if (params && 'id' in params && params.id === 'kql-telemetry:kql-telemetry') {
-      if (optCount === null) {
-        return Promise.resolve({
+  function mockedEsGetMethod() {
+    if (optCount === null) {
+      return Promise.resolve({
+        body: {
           _index: '.kibana_1',
           _id: 'kql-telemetry:kql-telemetry',
           found: false,
-        });
-      } else {
-        return Promise.resolve({
+        },
+      });
+    } else {
+      return Promise.resolve({
+        body: {
           _source: {
-            'kql-telemetry': {
-              ...optCount,
-            },
+            'kql-telemetry': { ...optCount },
             type: 'kql-telemetry',
             updated_at: '2018-10-05T20:20:56.258Z',
           },
-        });
-      }
-    } else if (params && 'body' in params && params.body.query.term.type === 'config') {
-      if (language === 'missingConfigDoc') {
-        return Promise.resolve({
-          hits: {
-            hits: [],
-          },
-        });
-      } else {
-        return Promise.resolve({
+        },
+      });
+    }
+  }
+  function mockedEsSearchMethod() {
+    if (language === 'missingConfigDoc') {
+      return Promise.resolve({ body: { hits: { hits: [] } } });
+    } else {
+      return Promise.resolve({
+        body: {
           hits: {
             hits: [
               {
@@ -77,12 +76,15 @@ function setupMockCallCluster(
               },
             ],
           },
-        });
-      }
+        },
+      });
     }
-
-    throw new Error('invalid call');
-  }) as unknown) as LegacyAPICaller;
+  }
+  const esClientMock = ({
+    get: jest.fn().mockImplementation(mockedEsGetMethod),
+    search: jest.fn().mockImplementation(mockedEsSearchMethod),
+  } as unknown) as ElasticsearchClient;
+  esClient = esClientMock;
 }
 
 describe('makeKQLUsageCollector', () => {
@@ -95,7 +97,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster({ optInCount: 1 }, 'kuery');
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.optInCount).toBe(1);
@@ -106,7 +108,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster({ optInCount: 1 }, 'kuery');
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.defaultQueryLanguage).toBe('kuery');
@@ -117,7 +119,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster({ optInCount: 1 }, null);
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.defaultQueryLanguage).toBe('lucene');
@@ -127,7 +129,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster({ optInCount: 1 }, undefined);
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.defaultQueryLanguage).toBe('default-lucene');
@@ -137,7 +139,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster(null, 'kuery');
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.optInCount).toBe(0);
@@ -148,7 +150,7 @@ describe('makeKQLUsageCollector', () => {
       setupMockCallCluster(null, 'missingConfigDoc');
       collectorFetchContext = {
         ...collectorFetchContextMock,
-        callCluster,
+        esClient,
       };
       const fetchResponse = await fetch(collectorFetchContext);
       expect(fetchResponse.defaultQueryLanguage).toBe('default-lucene');

src/plugins/data/server/kql_telemetry/usage_collector/fetch.ts

@@ -30,18 +30,22 @@ export interface Usage {
 }
 
 export function fetchProvider(index: string) {
-  return async ({ callCluster }: CollectorFetchContext): Promise<Usage> => {
-    const [response, config] = await Promise.all([
-      callCluster('get', {
-        index,
-        id: 'kql-telemetry:kql-telemetry',
-        ignore: [404],
-      }),
-      callCluster('search', {
-        index,
-        body: { query: { term: { type: 'config' } } },
-        ignore: [404],
-      }),
+  return async ({ esClient }: CollectorFetchContext): Promise<Usage> => {
+    const [{ body: response }, { body: config }] = await Promise.all([
+      esClient.get(
+        {
+          index,
+          id: 'kql-telemetry:kql-telemetry',
+        },
+        { ignore: [404] }
+      ),
+      esClient.search(
+        {
+          index,
+          body: { query: { term: { type: 'config' } } },
+        },
+        { ignore: [404] }
+      ),
     ]);
 
     const queryLanguageConfigValue: string | null | undefined = get(