[ML] AIOps: Identify spike/dips with change point detection for log rate analysis

x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx

@@ -19,13 +19,16 @@ import type {
   BarStyleAccessor,
   RectAnnotationSpec,
 } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
+
 import { getTimeZone } from '@kbn/visualization-utils';
 import { i18n } from '@kbn/i18n';
 import type { IUiSettingsClient } from '@kbn/core/public';
 import {
   getLogRateAnalysisType,
+  getSnappedTimestamps,
   getSnappedWindowParameters,
-  getWindowParameters,
+  getWindowParametersForTrigger,
+  type DocumentCountStatsChangePoint,
   type LogRateAnalysisType,
   type LogRateHistogramItem,
   type WindowParameters,
@@ -129,6 +132,8 @@ export interface DocumentCountChartProps {
   baselineBrush?: BrushSettings;
   /** Optional data-test-subject */
   dataTestSubj?: string;
+  /** Optional change point metadata */
+  changePoint?: DocumentCountStatsChangePoint;
 }
 
 const SPEC_ID = 'document_count';
@@ -163,6 +168,7 @@ function getBaselineBadgeOverflow(
  */
 export const DocumentCountChart: FC<DocumentCountChartProps> = (props) => {
   const {
+    changePoint,
     dataTestSubj,
     dependencies,
     brushSelectionUpdateHandler,
@@ -250,17 +256,10 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = (props) => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [chartPointsSplit, timeRangeEarliest, timeRangeLatest, interval]);
 
-  const snapTimestamps = useMemo(() => {
-    const timestamps: number[] = [];
-    let n = timeRangeEarliest;
-
-    while (n <= timeRangeLatest + interval) {
-      timestamps.push(n);
-      n += interval;
-    }
-
-    return timestamps;
-  }, [timeRangeEarliest, timeRangeLatest, interval]);
+  const snapTimestamps = useMemo(
+    () => getSnappedTimestamps(timeRangeEarliest, timeRangeLatest, interval),
+    [timeRangeEarliest, timeRangeLatest, interval]
+  );
 
   const timefilterUpdateHandler = useCallback(
     (range: TimeFilterRange) => {
@@ -306,14 +305,13 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = (props) => {
           windowParameters === undefined &&
           adjustedChartPoints !== undefined
         ) {
-          const wp =
-            typeof startRange === 'number'
-              ? getWindowParameters(
-                  startRange + interval / 2,
-                  timeRangeEarliest,
-                  timeRangeLatest + interval
-                )
-              : startRange;
+          const wp = getWindowParametersForTrigger(
+            startRange,
+            interval,
+            timeRangeEarliest,
+            timeRangeLatest,
+            changePoint
+          );
           const wpSnap = getSnappedWindowParameters(wp, snapTimestamps);
           setOriginalWindowParameters(wpSnap);
           setWindowParameters(wpSnap);
@@ -329,6 +327,7 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = (props) => {
       }
     },
     [
+      changePoint,
       interval,
       timeRangeEarliest,
       timeRangeLatest,

x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '@kbn/aiops-utils/types';
+import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types';
 
 export const filteredFrequentItemSets: ItemSet[] = [
   {

x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '@kbn/aiops-utils/types';
+import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types';
 
 export const frequentItemSets: ItemSet[] = [
   {

x-pack/packages/ml/aiops_test_utils/kibana_sample_data_logs/significant_terms.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const kibanaSampleDataLogsSignificantTermsBase = [
+  {
+    fieldName: 'agent.keyword',
+    fieldValue:
+      'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24',
+    pValue: '5.82e-12',
+  },
+  {
+    fieldName: 'clientip',
+    fieldValue: '30.156.16.164',
+    pValue: '2.81e-53',
+  },
+  {
+    fieldName: 'extension.keyword',
+    fieldValue: '',
+    pValue: '5.72e-12',
+  },
+  {
+    fieldName: 'geo.dest',
+    fieldValue: 'IN',
+    pValue: '8.35e-21',
+  },
+  {
+    fieldName: 'geo.srcdest',
+    fieldValue: 'US:IN',
+    pValue: '8.35e-21',
+  },
+  {
+    fieldName: 'host.keyword',
+    fieldValue: 'elastic-elastic-elastic.org',
+    pValue: '3.94e-45',
+  },
+  {
+    fieldName: 'ip',
+    fieldValue: '30.156.16.163',
+    pValue: '9.50e-54',
+  },
+  {
+    fieldName: 'machine.os.keyword',
+    fieldValue: 'win xp',
+    pValue: '4.25e-18',
+  },
+  {
+    fieldName: 'referer',
+    fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra',
+    pValue: '1.41e-53',
+  },
+  {
+    fieldName: 'response.keyword',
+    fieldValue: '404',
+    pValue: '2.10e-35',
+  },
+];

x-pack/packages/ml/aiops_utils/index.ts

@@ -5,11 +5,17 @@
  * 2.0.
  */
 
-export { getLogRateAnalysisType } from './get_log_rate_analysis_type';
-export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type';
-export { type LogRateHistogramItem } from './log_rate_histogram_item';
+export { LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR } from './log_rate_analysis/constants';
+export { getLogRateAnalysisType } from './log_rate_analysis/get_log_rate_analysis_type';
 export {
-  getSnappedWindowParameters,
-  getWindowParameters,
-  type WindowParameters,
-} from './window_parameters';
+  LOG_RATE_ANALYSIS_TYPE,
+  type LogRateAnalysisType,
+} from './log_rate_analysis/log_rate_analysis_type';
+export type { LogRateHistogramItem } from './log_rate_analysis/log_rate_histogram_item';
+export type { DocumentCountStatsChangePoint } from './log_rate_analysis/types';
+export type { WindowParameters } from './log_rate_analysis/window_parameters';
+export { getSnappedTimestamps } from './log_rate_analysis/get_snapped_timestamps';
+export { getSnappedWindowParameters } from './log_rate_analysis/get_snapped_window_parameters';
+export { getWindowParameters } from './log_rate_analysis/get_window_parameters';
+export { getWindowParametersForTrigger } from './log_rate_analysis/get_window_parameters_for_trigger';
+export { getExtendedChangePoint } from './log_rate_analysis/get_extended_change_point';

x-pack/packages/ml/aiops_utils/log_rate_analysis/__mocks__/date_histogram.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const getDateHistogramBuckets = (): Record<string, number> => ({
+  1654566600000: 4929,
+  1654566900000: 4686,
+  1654567200000: 5243,
+  1654567500000: 4186,
+  1654567800000: 5529,
+  1654568100000: 6071,
+  1654568400000: 4500,
+  1654568700000: 6157,
+  1654569000000: 4886,
+  1654569300000: 5886,
+  1654569600000: 4843,
+  1654569900000: 4871,
+  1654570200000: 5129,
+  1654570500000: 4529,
+  1654570800000: 5171,
+  1654571100000: 6357,
+  1654571400000: 4100,
+  1654571700000: 4714,
+  1654572000000: 5029,
+  1654572300000: 4100,
+  1654572600000: 5057,
+  1654572900000: 5129,
+  1654573200000: 4871,
+  1654573500000: 4914,
+  1654573800000: 4586,
+  1654574100000: 3857,
+  1654574400000: 3886,
+  1654574700000: 5286,
+  1654575000000: 4543,
+  1654575300000: 5800,
+  1654575600000: 4943,
+  1654575900000: 5071,
+  1654576200000: 6486,
+  1654576500000: 5914,
+  1654576800000: 5643,
+  1654577100000: 6500,
+  1654577400000: 7014,
+  1654577700000: 5300,
+  1654578000000: 6086,
+  1654578300000: 5829,
+  1654578600000: 6743,
+  1654578900000: 7457,
+  1654579200000: 5729,
+  1654579500000: 6871,
+  1654579800000: 7457,
+  1654580100000: 6657,
+  1654580400000: 8543,
+  1654580700000: 8629,
+  1654581000000: 8586,
+  1654581300000: 7043,
+  1654581600000: 8071,
+  1654581900000: 8471,
+  1654582200000: 12243,
+  1654582500000: 10171,
+  1654582800000: 10143,
+  1654583100000: 11529,
+  1654583400000: 10986,
+  1654583700000: 10757,
+  1654584000000: 12614,
+  1654584300000: 11771,
+  1654584600000: 11771,
+  1654584900000: 11543,
+  1654585200000: 10671,
+  1654585500000: 14914,
+  1654585800000: 12500,
+  1654586100000: 15029,
+  1654586400000: 99900,
+  1654586700000: 78971,
+  1654587000000: 20600,
+  1654587300000: 4300,
+  1654587600000: 11671,
+  1654587900000: 2629,
+  1654588200000: 2200,
+  1654588500000: 13157,
+  1654588800000: 2714,
+});

x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/** Highlighting color for charts */
+export const LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR = 'orange';

x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.test.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getDateHistogramBuckets } from './__mocks__/date_histogram';
+import { getExtendedChangePoint } from './get_extended_change_point';
+
+describe('getExtendedChangePoint', () => {
+  test('returns the extended change point', () => {
+    const changePointTs = 1654586400000;
+    expect(getExtendedChangePoint(getDateHistogramBuckets(), changePointTs)).toEqual({
+      endTs: 1654587000000,
+      startTs: 1654586100000,
+    });
+  });
+});

x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mean } from 'd3-array';
+
+/**
+ * Calculates and returns an extended change point range based on the specified change point timestamp.
+ *
+ * @param buckets - An object where keys are bucket timestamps as strings
+ *                  and values are numeric values associated with each bucket.
+ * @param changePointTs - The timestamp of the change point as a number. This timestamp must
+ *                        be one of the keys in the `buckets` object.
+ * @returns An object containing two properties: `startTs` and `endTs`.
+ */
+export const getExtendedChangePoint = (buckets: Record<string, number>, changePointTs: number) => {
+  const bucketKeys = Object.keys(buckets);
+  const bucketValues = Object.values(buckets);
+  const meanValue = Math.round(mean(bucketValues) ?? 0);
+  const cpIndex = bucketKeys.findIndex((d) => +d === changePointTs);
+  const cpValue = buckets[changePointTs];
+
+  let lIndex = cpIndex - 1;
+  let uIndex = cpIndex + 1;
+
+  while (
+    lIndex >= 0 &&
+    Math.abs(bucketValues[lIndex] - meanValue) > Math.abs(bucketValues[lIndex] - cpValue)
+  ) {
+    lIndex--;
+  }
+
+  while (
+    uIndex < bucketValues.length &&
+    Math.abs(bucketValues[uIndex] - meanValue) > Math.abs(bucketValues[uIndex] - cpValue)
+  ) {
+    uIndex++;
+  }
+
+  return { startTs: +bucketKeys[lIndex], endTs: +bucketKeys[uIndex] };
+};

x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Generates an array of timestamps evenly spaced within a given time range.
+ *
+ * @param timeRangeEarliest The earliest timestamp in the time range.
+ * @param timeRangeLatest The latest timestamp in the time range.
+ * @param interval The interval between timestamps in milliseconds.
+ * @returns Array of timestamps spaced by the specified interval within the given range.
+ */
+export const getSnappedTimestamps = (
+  timeRangeEarliest: number,
+  timeRangeLatest: number,
+  interval: number
+) => {
+  const timestamps: number[] = [];
+  let n = timeRangeEarliest;
+
+  while (n <= timeRangeLatest + interval) {
+    timestamps.push(n);
+    n += interval;
+  }
+
+  return timestamps;
+};