Merge branch 'main' into migrations/security_solution-timelines

src/plugins/expressions/common/expression_functions/specs/math_column.ts

@@ -14,6 +14,7 @@ import { Datatable, DatatableColumn, DatatableColumnType, getType } from '../../
 export type MathColumnArguments = MathArguments & {
   id: string;
   name?: string;
+  castColumns?: string[];
   copyMetaFrom?: string | null;
 };
 
@@ -52,6 +53,14 @@ export const mathColumn: ExpressionFunctionDefinition<
       }),
       required: true,
     },
+    castColumns: {
+      types: ['string'],
+      multi: true,
+      help: i18n.translate('expressions.functions.mathColumn.args.castColumnsHelpText', {
+        defaultMessage: 'The ids of columns to cast to numbers before applying the formula',
+      }),
+      required: false,
+    },
     copyMetaFrom: {
       types: ['string', 'null'],
       help: i18n.translate('expressions.functions.mathColumn.args.copyMetaFromHelpText', {
@@ -77,11 +86,31 @@ export const mathColumn: ExpressionFunctionDefinition<
 
     const newRows = await Promise.all(
       input.rows.map(async (row) => {
+        let preparedRow = row;
+        if (args.castColumns) {
+          preparedRow = { ...row };
+          args.castColumns.forEach((columnId) => {
+            switch (typeof row[columnId]) {
+              case 'string':
+                const parsedAsDate = Number(new Date(preparedRow[columnId]));
+                if (!isNaN(parsedAsDate)) {
+                  preparedRow[columnId] = parsedAsDate;
+                  return;
+                } else {
+                  preparedRow[columnId] = Number(preparedRow[columnId]);
+                  return;
+                }
+              case 'boolean':
+                preparedRow[columnId] = Number(preparedRow[columnId]);
+                return;
+            }
+          });
+        }
         const result = await math.fn(
           {
             ...input,
             columns: input.columns,
-            rows: [row],
+            rows: [preparedRow],
           },
           {
             expression: args.expression,

src/plugins/telemetry/common/constants.ts

@@ -12,6 +12,12 @@
  */
 export const REPORT_INTERVAL_MS = 86400000;
 
+/**
+ * The buffer time, in milliseconds, to consider the {@link REPORT_INTERVAL_MS} as expired.
+ * Currently, 2 minutes.
+ */
+export const REPORT_INTERVAL_BUFFER_MS = 120000;
+
 /**
  * How often we poll for the opt-in status.
  * Currently, 10 seconds.

src/plugins/telemetry/common/is_report_interval_expired.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { REPORT_INTERVAL_MS } from './constants';
+import { REPORT_INTERVAL_BUFFER_MS, REPORT_INTERVAL_MS } from './constants';
 import { isReportIntervalExpired } from './is_report_interval_expired';
 
 describe('isReportIntervalExpired', () => {
@@ -54,7 +54,9 @@ describe('isReportIntervalExpired', () => {
   });
 
   test('false when close but not yet', () => {
-    expect(isReportIntervalExpired(Date.now() - REPORT_INTERVAL_MS + 1000)).toBe(false);
+    expect(
+      isReportIntervalExpired(Date.now() - REPORT_INTERVAL_MS + REPORT_INTERVAL_BUFFER_MS + 1000)
+    ).toBe(false);
   });
 
   test('false when date in the future', () => {

src/plugins/telemetry/common/is_report_interval_expired.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { REPORT_INTERVAL_MS } from './constants';
+import { REPORT_INTERVAL_BUFFER_MS, REPORT_INTERVAL_MS } from './constants';
 
 /**
  * The report is considered expired if:
@@ -15,5 +15,9 @@ import { REPORT_INTERVAL_MS } from './constants';
  * @returns `true` if the report interval is considered expired
  */
 export function isReportIntervalExpired(lastReportAt: number | undefined) {
-  return !lastReportAt || isNaN(lastReportAt) || Date.now() - lastReportAt > REPORT_INTERVAL_MS;
+  return (
+    !lastReportAt ||
+    isNaN(lastReportAt) ||
+    Date.now() - lastReportAt > REPORT_INTERVAL_MS - REPORT_INTERVAL_BUFFER_MS
+  );
 }

src/plugins/telemetry/server/fetcher.test.mock.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const fetchMock = jest.fn();
+
+jest.doMock('node-fetch', () => fetchMock);
+
+export const getNextAttemptDateMock = jest.fn();
+
+jest.doMock('./get_next_attempt_date', () => ({ getNextAttemptDate: getNextAttemptDateMock }));

src/plugins/telemetry/server/fetcher.test.ts

@@ -7,14 +7,29 @@
  */
 
 /* eslint-disable dot-notation */
-import { FetcherTask } from './fetcher';
+import { fakeSchedulers } from 'rxjs-marbles/jest';
 import { coreMock } from '@kbn/core/server/mocks';
 import {
   telemetryCollectionManagerPluginMock,
   Setup,
 } from '@kbn/telemetry-collection-manager-plugin/server/mocks';
 
+jest.mock('rxjs', () => {
+  const RxJs = jest.requireActual('rxjs');
+  return {
+    ...RxJs,
+    // Redefining timer as a merge of timer and interval because `fakeSchedulers` fails to advance on the intervals
+    timer: (dueTime: number, interval: number) =>
+      RxJs.merge(RxJs.timer(dueTime), RxJs.interval(interval)),
+  };
+});
+
+import { fetchMock, getNextAttemptDateMock } from './fetcher.test.mock';
+import { FetcherTask } from './fetcher';
+
 describe('FetcherTask', () => {
+  beforeEach(() => jest.useFakeTimers('legacy'));
+
   describe('sendIfDue', () => {
     let getCurrentConfigs: jest.Mock;
     let shouldSendReport: jest.Mock;
@@ -95,4 +110,199 @@ describe('FetcherTask', () => {
       expect(updateReportFailure).toBeCalledTimes(0);
     });
   });
+
+  describe('Validate connectivity', () => {
+    let fetcherTask: FetcherTask;
+    let getCurrentConfigs: jest.Mock;
+    let updateReportFailure: jest.Mock;
+
+    beforeEach(() => {
+      getCurrentConfigs = jest.fn();
+      updateReportFailure = jest.fn();
+      fetcherTask = new FetcherTask(coreMock.createPluginInitializerContext({}));
+      Object.assign(fetcherTask, { getCurrentConfigs, updateReportFailure });
+    });
+
+    afterEach(() => {
+      fetchMock.mockReset();
+    });
+
+    test(
+      'Validates connectivity and sets as online when the OPTIONS request succeeds',
+      fakeSchedulers(async (advance) => {
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+        getCurrentConfigs.mockResolvedValue({
+          telemetryOptIn: true,
+          telemetrySendUsageFrom: 'server',
+          failureCount: 0,
+          telemetryUrl: 'test-url',
+        });
+        fetchMock.mockResolvedValue({});
+        const subscription = fetcherTask['validateConnectivity']();
+        advance(5 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(1);
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        expect(fetchMock).toHaveBeenCalledWith('test-url', { method: 'options' });
+        expect(fetcherTask['isOnline$'].value).toBe(true);
+        subscription.unsubscribe();
+      })
+    );
+
+    test(
+      'Skips validation when already set as online',
+      fakeSchedulers(async (advance) => {
+        fetcherTask['isOnline$'].next(true);
+        const subscription = fetcherTask['validateConnectivity']();
+        advance(5 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(0);
+        expect(fetchMock).toHaveBeenCalledTimes(0);
+        expect(fetcherTask['isOnline$'].value).toBe(true);
+        subscription.unsubscribe();
+      })
+    );
+
+    test(
+      'Retries on errors',
+      fakeSchedulers(async (advance) => {
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+        getCurrentConfigs.mockResolvedValue({
+          telemetryOptIn: true,
+          telemetrySendUsageFrom: 'server',
+          failureCount: 0,
+          telemetryUrl: 'test-url',
+        });
+        fetchMock.mockRejectedValue(new Error('Something went terribly wrong'));
+        const subscription = fetcherTask['validateConnectivity']();
+        advance(5 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(1);
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        expect(updateReportFailure).toHaveBeenCalledTimes(1);
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+
+        // Try again after 12 hours
+        fetchMock.mockResolvedValue({});
+        advance(12 * 60 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(2);
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+        expect(updateReportFailure).toHaveBeenCalledTimes(1);
+        expect(fetcherTask['isOnline$'].value).toBe(true);
+
+        subscription.unsubscribe();
+      })
+    );
+
+    test(
+      'Should not retry if it hit the max number of failures for this version',
+      fakeSchedulers(async (advance) => {
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+        getCurrentConfigs.mockResolvedValue({
+          telemetryOptIn: true,
+          telemetrySendUsageFrom: 'server',
+          failureCount: 3,
+          failureVersion: 'version',
+          currentVersion: 'version',
+          telemetryUrl: 'test-url',
+        });
+        const subscription = fetcherTask['validateConnectivity']();
+        advance(5 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(1);
+        expect(fetchMock).toHaveBeenCalledTimes(0);
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+
+        subscription.unsubscribe();
+      })
+    );
+
+    test(
+      'Should retry if it hit the max number of failures for a different version',
+      fakeSchedulers(async (advance) => {
+        expect(fetcherTask['isOnline$'].value).toBe(false);
+        getCurrentConfigs.mockResolvedValue({
+          telemetryOptIn: true,
+          telemetrySendUsageFrom: 'server',
+          failureCount: 3,
+          failureVersion: 'version',
+          currentVersion: 'another_version',
+          telemetryUrl: 'test-url',
+        });
+        const subscription = fetcherTask['validateConnectivity']();
+        advance(5 * 60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve)); // Wait for the promise to fulfill
+        expect(getCurrentConfigs).toHaveBeenCalledTimes(1);
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        expect(fetcherTask['isOnline$'].value).toBe(true);
+
+        subscription.unsubscribe();
+      })
+    );
+  });
+
+  describe('startSendIfDueSubscription', () => {
+    let fetcherTask: FetcherTask;
+    let sendIfDue: jest.Mock;
+
+    beforeEach(() => {
+      sendIfDue = jest.fn().mockResolvedValue({});
+      fetcherTask = new FetcherTask(coreMock.createPluginInitializerContext({}));
+      Object.assign(fetcherTask, { sendIfDue });
+    });
+
+    afterEach(() => {
+      getNextAttemptDateMock.mockReset();
+    });
+
+    test('Tries to send telemetry when it is online', () => {
+      const subscription = fetcherTask['startSendIfDueSubscription']();
+      fetcherTask['isOnline$'].next(true);
+      expect(sendIfDue).toHaveBeenCalledTimes(1);
+      subscription.unsubscribe();
+    });
+
+    test('Does not send telemetry when it is offline', () => {
+      const subscription = fetcherTask['startSendIfDueSubscription']();
+      fetcherTask['isOnline$'].next(false);
+      expect(sendIfDue).toHaveBeenCalledTimes(0);
+      subscription.unsubscribe();
+    });
+
+    test(
+      'Sends telemetry when the next attempt date kicks in',
+      fakeSchedulers((advance) => {
+        fetcherTask['isOnline$'].next(true);
+        const subscription = fetcherTask['startSendIfDueSubscription']();
+        const lastReported = Date.now();
+        getNextAttemptDateMock.mockReturnValue(new Date(lastReported + 1000));
+        fetcherTask['lastReported$'].next(lastReported);
+        advance(1000);
+        expect(sendIfDue).toHaveBeenCalledTimes(1);
+        subscription.unsubscribe();
+      })
+    );
+
+    test(
+      'Keeps retrying every 1 minute after the next attempt date until a new emission of lastReported occurs',
+      fakeSchedulers(async (advance) => {
+        fetcherTask['isOnline$'].next(true);
+        const subscription = fetcherTask['startSendIfDueSubscription']();
+        const lastReported = Date.now();
+        getNextAttemptDateMock.mockReturnValue(new Date(lastReported + 1000));
+        fetcherTask['lastReported$'].next(lastReported);
+        advance(1000);
+        await new Promise((resolve) => process.nextTick(resolve));
+        expect(sendIfDue).toHaveBeenCalledTimes(1);
+        advance(60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve));
+        expect(sendIfDue).toHaveBeenCalledTimes(2);
+        advance(60 * 1000);
+        await new Promise((resolve) => process.nextTick(resolve));
+        expect(sendIfDue).toHaveBeenCalledTimes(3);
+        subscription.unsubscribe();
+      })
+    );
+  });
 });