Merge remote-tracking branch 'elastic' into feature/canvas-snapshots

.github/CODEOWNERS

@@ -35,6 +35,7 @@
 /x-pack/test/functional/apps/machine_learning/  @elastic/ml-ui
 /x-pack/test/functional/services/machine_learning/  @elastic/ml-ui
 /x-pack/test/functional/services/ml.ts  @elastic/ml-ui
+/x-pack/legacy/plugins/transform/  @elastic/ml-ui
 
 # Operations
 /renovate.json5 @elastic/kibana-operations

docs/maps/maps-aggregations.asciidoc

@@ -36,8 +36,6 @@ To enable most recent entities, click "Show most recent documents by entity" and
 
 . Set *Entity* to the field that identifies entities in your documents.
 This field will be used in the terms aggregation to group your documents into entity buckets.
-. Set *Time* to the date field that puts your documents in chronological order.
-This field will be used to sort your documents in the top hits aggregation.
 . Set *Documents per entity* to configure the maximum number of documents accumulated per entity.
 
 [role="xpack"]

docs/user/security/authentication/index.asciidoc

@@ -100,7 +100,7 @@ xpack.security.authc.saml.realm: realm-name
 +
 [source,yaml]
 --------------------------------------------------------------------------------
-server.xsrf.whitelist: [/api/security/v1/saml]
+server.xsrf.whitelist: [/api/security/saml/callback]
 --------------------------------------------------------------------------------
 
 Users will be able to log in to {kib} via SAML Single Sign-On by navigating directly to the {kib} URL. Users who aren't authenticated are redirected to the Identity Provider for login. Most Identity Providers maintain a long-lived session—users who logged in to a different application using the same Identity Provider in the same browser are automatically authenticated. An exception is if {es} or the Identity Provider is configured to force user to re-authenticate. This login scenario is called _Service Provider initiated login_.
@@ -119,6 +119,21 @@ The order of `saml` and `basic` is important. Users who open {kib} will go throu
 
 Basic authentication is supported _only_ if `basic` authentication provider is explicitly declared in `xpack.security.authc.providers` setting in addition to `saml`.
 
+[float]
+===== SAML and long URLs
+
+At the beginning of the SAML handshake, {kib} stores the initial URL in the session cookie, so it can redirect the user back to that URL after successful SAML authentication.
+If the URL is long, the session cookie might exceed the maximum size supported by the browser--typically 4KB for all cookies per domain. When this happens, the session cookie is truncated,
+or dropped completely, and you might experience sporadic failures during SAML authentication. 
+
+To remedy this issue, you can decrease the maximum
+size of the URL that {kib} is allowed to store during the SAML handshake. The default value is 2KB.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+xpack.security.authc.saml.maxRedirectURLSize: 1kb
+--------------------------------------------------------------------------------
+
 [[oidc]]
 ==== OpenID Connect Single Sign-On
 

package.json

@@ -70,7 +70,8 @@
     "core:acceptApiChanges": "node scripts/check_core_api_changes.js --accept",
     "kbn:bootstrap": "yarn build:types && node scripts/register_git_hook",
     "spec_to_console": "node scripts/spec_to_console",
-    "backport-skip-ci": "backport --prDescription \"[skip-ci]\""
+    "backport-skip-ci": "backport --prDescription \"[skip-ci]\"",
+    "cover:report": "nyc report --temp-dir target/kibana-coverage/functional --report-dir target/coverage/report --reporter=lcov && open ./target/coverage/report/lcov-report/index.html"
   },
   "repository": {
     "type": "git",
@@ -105,7 +106,7 @@
   "dependencies": {
     "@babel/core": "^7.5.5",
     "@babel/register": "^7.5.5",
-    "@elastic/charts": "^13.4.0",
+    "@elastic/charts": "^13.5.1",
     "@elastic/datemath": "5.0.2",
     "@elastic/eui": "14.4.0",
     "@elastic/filesaver": "1.1.2",
@@ -256,11 +257,12 @@
     "vega-lib": "4.3.0",
     "vega-lite": "^2.6.0",
     "vega-schema-url-parser": "1.0.0",
-    "vega-tooltip": "^0.9.14",
+    "vega-tooltip": "^0.19.1",
     "vision": "^5.3.3",
     "webpack": "4.41.0",
     "webpack-merge": "4.2.2",
     "whatwg-fetch": "^3.0.0",
+    "wrapper-webpack-plugin": "^2.1.0",
     "yauzl": "2.10.0"
   },
   "devDependencies": {
@@ -303,7 +305,7 @@
     "@types/getopts": "^2.0.1",
     "@types/glob": "^7.1.1",
     "@types/globby": "^8.0.0",
-    "@types/graphql": "^0.13.1",
+    "@types/graphql": "^0.13.2",
     "@types/hapi": "^17.0.18",
     "@types/hapi-auth-cookie": "^9.1.0",
     "@types/has-ansi": "^3.0.0",
@@ -355,6 +357,7 @@
     "babel-eslint": "^10.0.3",
     "babel-jest": "^24.9.0",
     "babel-plugin-dynamic-import-node": "^2.3.0",
+    "babel-plugin-istanbul": "^5.2.0",
     "backport": "4.7.1",
     "chai": "3.5.0",
     "chance": "1.0.18",

packages/kbn-babel-preset/istanbul_preset.js

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+module.exports = () => {
+  return {
+    plugins: ['istanbul']
+  };
+};

packages/kbn-test/src/failed_tests_reporter/github_api.ts

@@ -21,7 +21,7 @@ import Url from 'url';
 
 import Axios, { AxiosRequestConfig } from 'axios';
 import parseLinkHeader from 'parse-link-header';
-import { ToolingLog, isAxiosResponseError } from '@kbn/dev-utils';
+import { ToolingLog, isAxiosResponseError, isAxiosRequestError } from '@kbn/dev-utils';
 
 const BASE_URL = 'https://api.github.com/repos/elastic/kibana/';
 
@@ -33,7 +33,11 @@ export interface GithubIssue {
   body: string;
 }
 
-type RequestOptions = AxiosRequestConfig & { safeForDryRun?: boolean };
+type RequestOptions = AxiosRequestConfig & {
+  safeForDryRun?: boolean;
+  maxAttempts?: number;
+  attempt?: number;
+};
 
 export class GithubApi {
   private readonly x = Axios.create({
@@ -78,7 +82,8 @@ export class GithubApi {
         issues.push(issue);
       }
 
-      const parsed = parseLinkHeader(resp.headers.link);
+      const parsed =
+        typeof resp.headers.link === 'string' ? parseLinkHeader(resp.headers.link) : undefined;
       if (parsed && parsed.next && parsed.next.url) {
         nextRequest = {
           safeForDryRun: true,
@@ -139,31 +144,64 @@ export class GithubApi {
     return resp.data.html_url;
   }
 
-  private async request<T>(options: RequestOptions, dryRunResponse: T) {
+  private async request<T>(
+    options: RequestOptions,
+    dryRunResponse: T
+  ): Promise<{
+    status: number;
+    statusText: string;
+    headers: Record<string, string | string[] | undefined>;
+    data: T;
+  }> {
     const executeRequest = !this.dryRun || options.safeForDryRun;
+    const maxAttempts = options.maxAttempts || 5;
+    const attempt = options.attempt || 1;
+
     this.log.verbose('Github API', executeRequest ? 'Request' : 'Dry Run', options);
 
-    if (executeRequest) {
-      try {
-        return await this.x.request<T>(options);
-      } catch (error) {
-        if (isAxiosResponseError(error)) {
-          throw new Error(
-            `[${error.config.method} ${error.config.url}] ${error.response.status} ${
-              error.response.statusText
-            } Error: ${JSON.stringify(error.response.data)}`
-          );
+    if (!executeRequest) {
+      return {
+        status: 200,
+        statusText: 'OK',
+        headers: {},
+        data: dryRunResponse,
+      };
+    }
+
+    try {
+      return await this.x.request<T>(options);
+    } catch (error) {
+      const unableToReachGithub = isAxiosRequestError(error);
+      const githubApiFailed = isAxiosResponseError(error) && error.response.status >= 500;
+      const errorResponseLog =
+        isAxiosResponseError(error) &&
+        `[${error.config.method} ${error.config.url}] ${error.response.status} ${error.response.statusText} Error`;
+
+      if ((unableToReachGithub || githubApiFailed) && attempt < maxAttempts) {
+        const waitMs = 1000 * attempt;
+
+        if (errorResponseLog) {
+          this.log.error(`${errorResponseLog}: waiting ${waitMs}ms to retry`);
+        } else {
+          this.log.error(`Unable to reach github, waiting ${waitMs}ms to retry`);
         }
 
-        throw error;
+        await new Promise(resolve => setTimeout(resolve, waitMs));
+        return await this.request<T>(
+          {
+            ...options,
+            maxAttempts,
+            attempt: attempt + 1,
+          },
+          dryRunResponse
+        );
       }
-    }
 
-    return {
-      status: 200,
-      statusText: 'OK',
-      headers: {},
-      data: dryRunResponse,
-    };
+      if (errorResponseLog) {
+        throw new Error(`${errorResponseLog}: ${JSON.stringify(error.response.data)}`);
+      }
+
+      throw error;
+    }
   }
 }

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -36,7 +36,7 @@ export function createLifecycle() {
     phaseEnd: [] as Listener[],
   };
 
-  const cleanup$ = new Rx.ReplaySubject(1);
+  const cleanup$ = new Rx.ReplaySubject<undefined>(1);
 
   return {
     cleanup$: cleanup$.asObservable(),

renovate.json5

@@ -815,6 +815,22 @@
           '@types/tinycolor2',
         ],
       },
+      {
+        groupSlug: 'xml2js',
+        groupName: 'xml2js related packages',
+        packageNames: [
+          'xml2js',
+          '@types/xml2js',
+        ],
+      },
+      {
+        groupSlug: 'xml-crypto',
+        groupName: 'xml-crypto related packages',
+        packageNames: [
+          'xml-crypto',
+          '@types/xml-crypto',
+        ],
+      },
       {
         groupSlug: 'intl-relativeformat',
         groupName: 'intl-relativeformat related packages',
@@ -927,14 +943,6 @@
           '@types/parse-link-header',
         ],
       },
-      {
-        groupSlug: 'xml2js',
-        groupName: 'xml2js related packages',
-        packageNames: [
-          'xml2js',
-          '@types/xml2js',
-        ],
-      },
       {
         packagePatterns: [
           '^@kbn/.*',

src/core/server/legacy/logging/legacy_logging_server.test.ts

@@ -50,7 +50,7 @@ test('correctly forwards log records.', () => {
     level: LogLevel.Trace,
     context: 'some-context.sub-context',
     message: 'some-message',
-    meta: { tags: ['important', 'tags'] },
+    meta: { tags: ['important', 'tags'], unknown: 2 },
   };
 
   loggingServer.log(firstLogRecord);
@@ -85,7 +85,14 @@ Object {
 
   expect(thirdCall).toMatchInlineSnapshot(`
 Object {
-  "data": "some-message",
+  "data": Object {
+    Symbol(log message with metadata): Object {
+      "message": "some-message",
+      "metadata": Object {
+        "unknown": 2,
+      },
+    },
+  },
   "tags": Array [
     "debug",
     "some-context",

src/core/server/legacy/logging/legacy_logging_server.ts

@@ -26,6 +26,23 @@ import { setupLogging } from '../../../../legacy/server/logging';
 import { LogLevel } from '../../logging/log_level';
 import { LogRecord } from '../../logging/log_record';
 
+export const metadataSymbol = Symbol('log message with metadata');
+export function attachMetaData(message: string, metadata: Record<string, any> = {}) {
+  return {
+    [metadataSymbol]: {
+      message,
+      metadata,
+    },
+  };
+}
+const isEmptyObject = (obj: object) => Object.keys(obj).length === 0;
+
+function getDataToLog(error: Error | undefined, metadata: object, message: string) {
+  if (error) return error;
+  if (!isEmptyObject(metadata)) return attachMetaData(message, metadata);
+  return message;
+}
+
 interface PluginRegisterParams {
   plugin: {
     register: (
@@ -90,9 +107,11 @@ export class LegacyLoggingServer {
   }
 
   public log({ level, context, message, error, timestamp, meta = {} }: LogRecord) {
+    const { tags = [], ...metadata } = meta;
+
     this.events.emit('log', {
-      data: error || message,
-      tags: [getLegacyLogLevel(level), ...context.split('.'), ...(meta.tags || [])],
+      data: getDataToLog(error, metadata, message),
+      tags: [getLegacyLogLevel(level), ...context.split('.'), ...tags],
       timestamp: timestamp.getTime(),
     });
   }

src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker

@@ -174,6 +174,7 @@ kibana_vars=(
     xpack.security.authc.providers
     xpack.security.authc.oidc.realm
     xpack.security.authc.saml.realm
+    xpack.security.authc.saml.maxRedirectURLSize
     xpack.security.cookieName
     xpack.security.enabled
     xpack.security.encryptionKey