From 9bd28052cfe5a6db57e4e342ea540705649ee2d5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 8 Mar 2024 18:10:57 +0100 Subject: [PATCH 01/17] identify spike/dips with change point detection --- .../ml/aiops_utils/window_parameters.ts | 7 ++- .../log_rate_analysis_content.tsx | 19 ++++++ .../aiops/public/get_document_stats.ts | 59 ++++++++++++++++++- .../public/hooks/use_document_count_stats.ts | 4 +- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/x-pack/packages/ml/aiops_utils/window_parameters.ts b/x-pack/packages/ml/aiops_utils/window_parameters.ts index 91ecee5f841a2..7b60808226784 100644 --- a/x-pack/packages/ml/aiops_utils/window_parameters.ts +++ b/x-pack/packages/ml/aiops_utils/window_parameters.ts @@ -55,7 +55,8 @@ export const isWindowParameters = (arg: unknown): arg is WindowParameters => export const getWindowParameters = ( clickTime: number, minTime: number, - maxTime: number + maxTime: number, + clickTimeUpper?: number ): WindowParameters => { const totalWindow = maxTime - minTime; @@ -72,8 +73,8 @@ export const getWindowParameters = ( const baselineWindow = Math.max(totalWindow / 3.5, minBaselineWindow); const windowGap = Math.max(totalWindow / 10, minWindowGap); - const deviationMin = clickTime - deviationWindow / 2; - const deviationMax = clickTime + deviationWindow / 2; + const deviationMin = clickTimeUpper ? clickTime : clickTime - deviationWindow / 2; + const deviationMax = clickTimeUpper ? clickTimeUpper : clickTime + deviationWindow / 2; const baselineMax = deviationMin - windowGap; const baselineMin = baselineMax - baselineWindow; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 1ed80f54be2a6..28ddd69e5e905 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -11,6 +11,7 @@ import { EuiEmptyPrompt, EuiHorizontalRule, EuiPanel } from '@elastic/eui'; import type { Moment } from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { BarStyleAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -181,6 +182,23 @@ export const LogRateAnalysisContent: FC = ({ setInitialAnalysisStart(undefined); } + const barStyle = { + rect: { + opacity: 1, + fill: 'orange', + }, + }; + const barStyleAccessor: BarStyleAccessor | undefined = documentCountStats?.changePoint + ? (d, g) => { + return g.specId === 'document_count' && + documentCountStats?.changePoint && + d.x > documentCountStats.changePoint.lower && + d.x < documentCountStats.changePoint.upper + ? barStyle + : null; + } + : undefined; + return ( {documentCountStats !== undefined && ( @@ -198,6 +216,7 @@ export const LogRateAnalysisContent: FC = ({ initialAnalysisStart={initialAnalysisStart} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + barStyleAccessor={barStyleAccessor} /> )} diff --git a/x-pack/plugins/aiops/public/get_document_stats.ts b/x-pack/plugins/aiops/public/get_document_stats.ts index ad0b557d3a69e..ac25c44f0e2dc 100644 --- a/x-pack/plugins/aiops/public/get_document_stats.ts +++ b/x-pack/plugins/aiops/public/get_document_stats.ts @@ -6,6 +6,7 @@ */ import { get } from 'lodash'; +import { mean } from 'd3-array'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import dateMath from '@kbn/datemath'; @@ -21,6 +22,7 @@ import type { GroupTableItem } from './components/log_rate_analysis_results_tabl export interface DocumentCountStats { interval?: number; buckets?: { [key: string]: number }; + changePoint?: { key: number; lower: number; upper: number; type: string }; timeRangeEarliest?: number; timeRangeLatest?: number; totalCount: number; @@ -45,7 +47,8 @@ export interface DocumentStatsSearchStrategyParams { export const getDocumentCountStatsRequest = ( params: DocumentStatsSearchStrategyParams, randomSamplerWrapper?: RandomSamplerWrapper, - skipAggs = false + skipAggs = false, + changePoints = false ) => { const { index, @@ -88,6 +91,16 @@ export const getDocumentCountStatsRequest = ( : {}), }, }, + ...(changePoints + ? { + change_point_request: { + // @ts-expect-error missing from ES spec + change_point: { + buckets_path: 'eventRate>_count', + }, + }, + } + : {}), }; const aggs = randomSamplerWrapper ? randomSamplerWrapper.wrap(rawAggs) : rawAggs; @@ -152,11 +165,46 @@ export const processDocumentCountStats = ( [] ); + const changePointRaw = get( + randomSamplerWrapper && body.aggregations !== undefined + ? randomSamplerWrapper.unwrap(body.aggregations) + : body.aggregations, + ['change_point_request'] + ); + + const changePointBase = + changePointRaw && changePointRaw.bucket && Object.keys(changePointRaw.type).length > 0 + ? { key: Date.parse(changePointRaw.bucket.key), type: Object.keys(changePointRaw.type)[0] } + : undefined; + const buckets = dataByTimeBucket.reduce>((acc, cur) => { acc[cur.key] = cur.doc_count; return acc; }, {}); + const bucketKeys = Object.keys(buckets); + const bucketValues = Object.values(buckets); + const meanValue = Math.round(mean(bucketValues) ?? 0); + const cpIndex = bucketKeys.findIndex((d) => +d === changePointBase?.key); + const cpValue = changePointBase ? buckets[changePointBase.key] : 0; + + let lIndex = cpIndex - 1; + let uIndex = cpIndex + 1; + + while ( + lIndex >= 0 && + Math.abs(bucketValues[lIndex] - meanValue) > Math.abs(bucketValues[lIndex] - cpValue) + ) { + lIndex--; + } + + while ( + uIndex < bucketValues.length && + Math.abs(bucketValues[uIndex] - meanValue) > Math.abs(bucketValues[uIndex] - cpValue) + ) { + uIndex++; + } + const lastDocTimeStamp: string = Object.values(body.hits.hits[0]?.fields ?? [[]])[0][0]; const lastDocTimeStampMs = lastDocTimeStamp === undefined ? undefined : dateMath.parse(lastDocTimeStamp)?.valueOf(); @@ -168,5 +216,14 @@ export const processDocumentCountStats = ( timeRangeLatest: params.latest, totalCount, lastDocTimeStampMs, + ...(changePointBase + ? { + changePoint: { + ...changePointBase, + lower: +bucketKeys[lIndex], + upper: +bucketKeys[uIndex], + }, + } + : {}), }; }; diff --git a/x-pack/plugins/aiops/public/hooks/use_document_count_stats.ts b/x-pack/plugins/aiops/public/hooks/use_document_count_stats.ts index 54412c9d73877..d7fd3db593382 100644 --- a/x-pack/plugins/aiops/public/hooks/use_document_count_stats.ts +++ b/x-pack/plugins/aiops/public/hooks/use_document_count_stats.ts @@ -115,7 +115,9 @@ export function useDocumentCountStats Date: Tue, 12 Mar 2024 13:56:51 +0100 Subject: [PATCH 02/17] deviation selection based on change point detection --- .../document_count_chart.tsx | 52 ++++++++++++++++--- x-pack/packages/ml/aiops_utils/index.ts | 1 + x-pack/packages/ml/aiops_utils/types.ts | 7 +++ .../ml/aiops_utils/window_parameters.ts | 5 +- .../document_count_content.tsx | 1 + .../aiops/public/get_document_stats.ts | 5 +- 6 files changed, 59 insertions(+), 12 deletions(-) diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index 3c8f3c46bb866..2120502846295 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -19,6 +19,7 @@ import type { BarStyleAccessor, RectAnnotationSpec, } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; + import { getTimeZone } from '@kbn/visualization-utils'; import { i18n } from '@kbn/i18n'; import type { IUiSettingsClient } from '@kbn/core/public'; @@ -26,6 +27,7 @@ import { getLogRateAnalysisType, getSnappedWindowParameters, getWindowParameters, + type DocumentCountStatsChangePoint, type LogRateAnalysisType, type LogRateHistogramItem, type WindowParameters, @@ -39,6 +41,37 @@ import { DualBrush, DualBrushAnnotation } from '../..'; import { BrushBadge } from './brush_badge'; +function getWindowParametersForTrigger( + startRange: number | WindowParameters, + interval: number, + timeRangeEarliest: number, + timeRangeLatest: number, + changePoint?: DocumentCountStatsChangePoint +) { + if ( + typeof startRange === 'number' && + changePoint && + startRange >= changePoint.lower && + startRange <= changePoint.upper + ) { + return getWindowParameters( + changePoint.lower + interval, + timeRangeEarliest, + timeRangeLatest + interval, + changePoint.upper, + interval + ); + } else if (typeof startRange === 'number') { + return getWindowParameters( + startRange + interval / 2, + timeRangeEarliest, + timeRangeLatest + interval + ); + } + + return startRange; +} + declare global { interface Window { /** @@ -129,6 +162,8 @@ export interface DocumentCountChartProps { baselineBrush?: BrushSettings; /** Optional data-test-subject */ dataTestSubj?: string; + /** Optional change point metadata */ + changePoint?: DocumentCountStatsChangePoint; } const SPEC_ID = 'document_count'; @@ -163,6 +198,7 @@ function getBaselineBadgeOverflow( */ export const DocumentCountChart: FC = (props) => { const { + changePoint, dataTestSubj, dependencies, brushSelectionUpdateHandler, @@ -306,14 +342,13 @@ export const DocumentCountChart: FC = (props) => { windowParameters === undefined && adjustedChartPoints !== undefined ) { - const wp = - typeof startRange === 'number' - ? getWindowParameters( - startRange + interval / 2, - timeRangeEarliest, - timeRangeLatest + interval - ) - : startRange; + const wp = getWindowParametersForTrigger( + startRange, + interval, + timeRangeEarliest, + timeRangeLatest, + changePoint + ); const wpSnap = getSnappedWindowParameters(wp, snapTimestamps); setOriginalWindowParameters(wpSnap); setWindowParameters(wpSnap); @@ -329,6 +364,7 @@ export const DocumentCountChart: FC = (props) => { } }, [ + changePoint, interval, timeRangeEarliest, timeRangeLatest, diff --git a/x-pack/packages/ml/aiops_utils/index.ts b/x-pack/packages/ml/aiops_utils/index.ts index 6d22ce5849ec6..232dad3303237 100644 --- a/x-pack/packages/ml/aiops_utils/index.ts +++ b/x-pack/packages/ml/aiops_utils/index.ts @@ -8,6 +8,7 @@ export { getLogRateAnalysisType } from './get_log_rate_analysis_type'; export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; export { type LogRateHistogramItem } from './log_rate_histogram_item'; +export { type DocumentCountStatsChangePoint } from './types'; export { getSnappedWindowParameters, getWindowParameters, diff --git a/x-pack/packages/ml/aiops_utils/types.ts b/x-pack/packages/ml/aiops_utils/types.ts index 3d7bb6d15ef27..f29869c3b4f73 100644 --- a/x-pack/packages/ml/aiops_utils/types.ts +++ b/x-pack/packages/ml/aiops_utils/types.ts @@ -44,3 +44,10 @@ export interface SimpleHierarchicalTreeNode { children: SimpleHierarchicalTreeNode[]; addNode: (node: SimpleHierarchicalTreeNode) => void; } + +export interface DocumentCountStatsChangePoint { + key: number; + lower: number; + upper: number; + type: string; +} diff --git a/x-pack/packages/ml/aiops_utils/window_parameters.ts b/x-pack/packages/ml/aiops_utils/window_parameters.ts index 7b60808226784..26e608d8852e1 100644 --- a/x-pack/packages/ml/aiops_utils/window_parameters.ts +++ b/x-pack/packages/ml/aiops_utils/window_parameters.ts @@ -56,7 +56,8 @@ export const getWindowParameters = ( clickTime: number, minTime: number, maxTime: number, - clickTimeUpper?: number + clickTimeUpper?: number, + windowGapOverride?: number ): WindowParameters => { const totalWindow = maxTime - minTime; @@ -71,7 +72,7 @@ export const getWindowParameters = ( // being 3.5/10 of the total window. const deviationWindow = Math.max(totalWindow / 10, minDeviationWindow); const baselineWindow = Math.max(totalWindow / 3.5, minBaselineWindow); - const windowGap = Math.max(totalWindow / 10, minWindowGap); + const windowGap = windowGapOverride ?? Math.max(totalWindow / 10, minWindowGap); const deviationMin = clickTimeUpper ? clickTime : clickTime - deviationWindow / 2; const deviationMax = clickTimeUpper ? clickTimeUpper : clickTime + deviationWindow / 2; diff --git a/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx b/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx index 396ea54080749..951b9af635ca5 100644 --- a/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx @@ -113,6 +113,7 @@ export const DocumentCountContent: FC = ({ autoAnalysisStart={initialAnalysisStart} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + changePoint={documentCountStats.changePoint} {...docCountChartProps} /> diff --git a/x-pack/plugins/aiops/public/get_document_stats.ts b/x-pack/plugins/aiops/public/get_document_stats.ts index ac25c44f0e2dc..8e1417508f968 100644 --- a/x-pack/plugins/aiops/public/get_document_stats.ts +++ b/x-pack/plugins/aiops/public/get_document_stats.ts @@ -9,8 +9,9 @@ import { get } from 'lodash'; import { mean } from 'd3-array'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import dateMath from '@kbn/datemath'; +import dateMath from '@kbn/datemath'; +import type { DocumentCountStatsChangePoint } from '@kbn/aiops-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import type { SignificantItem } from '@kbn/ml-agg-utils'; import type { Query } from '@kbn/es-query'; @@ -22,7 +23,7 @@ import type { GroupTableItem } from './components/log_rate_analysis_results_tabl export interface DocumentCountStats { interval?: number; buckets?: { [key: string]: number }; - changePoint?: { key: number; lower: number; upper: number; type: string }; + changePoint?: DocumentCountStatsChangePoint; timeRangeEarliest?: number; timeRangeLatest?: number; totalCount: number; From 8c690aef6e56f5bcaeee30c86b1aa291b5147073 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Mar 2024 13:40:07 +0100 Subject: [PATCH 03/17] refactor window parameters --- .../document_count_chart.tsx | 33 +--- .../aiops_utils/get_log_rate_analysis_type.ts | 2 +- x-pack/packages/ml/aiops_utils/index.ts | 13 +- .../get_snapped_window_parameters.ts | 68 ++++++++ .../get_window_parameters.ts | 61 +++++++ .../get_window_parameters_for_trigger.ts | 41 +++++ .../log_rate_analysis/window_parameters.ts | 36 +++++ .../ml/aiops_utils/window_parameters.ts | 149 ------------------ 8 files changed, 214 insertions(+), 189 deletions(-) create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.ts create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/window_parameters.ts delete mode 100644 x-pack/packages/ml/aiops_utils/window_parameters.ts diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index 2120502846295..1e3c412eca173 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -26,7 +26,7 @@ import type { IUiSettingsClient } from '@kbn/core/public'; import { getLogRateAnalysisType, getSnappedWindowParameters, - getWindowParameters, + getWindowParametersForTrigger, type DocumentCountStatsChangePoint, type LogRateAnalysisType, type LogRateHistogramItem, @@ -41,37 +41,6 @@ import { DualBrush, DualBrushAnnotation } from '../..'; import { BrushBadge } from './brush_badge'; -function getWindowParametersForTrigger( - startRange: number | WindowParameters, - interval: number, - timeRangeEarliest: number, - timeRangeLatest: number, - changePoint?: DocumentCountStatsChangePoint -) { - if ( - typeof startRange === 'number' && - changePoint && - startRange >= changePoint.lower && - startRange <= changePoint.upper - ) { - return getWindowParameters( - changePoint.lower + interval, - timeRangeEarliest, - timeRangeLatest + interval, - changePoint.upper, - interval - ); - } else if (typeof startRange === 'number') { - return getWindowParameters( - startRange + interval / 2, - timeRangeEarliest, - timeRangeLatest + interval - ); - } - - return startRange; -} - declare global { interface Window { /** diff --git a/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts b/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts index 0303d903da05b..dc17d720bc42c 100644 --- a/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts +++ b/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts @@ -9,7 +9,7 @@ import { median } from 'd3-array'; import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; import type { LogRateHistogramItem } from './log_rate_histogram_item'; -import type { WindowParameters } from './window_parameters'; +import type { WindowParameters } from './log_rate_analysis/window_parameters'; /** * Identify the log rate analysis type based on the baseline/deviation diff --git a/x-pack/packages/ml/aiops_utils/index.ts b/x-pack/packages/ml/aiops_utils/index.ts index 232dad3303237..70b33c6d5fa45 100644 --- a/x-pack/packages/ml/aiops_utils/index.ts +++ b/x-pack/packages/ml/aiops_utils/index.ts @@ -7,10 +7,9 @@ export { getLogRateAnalysisType } from './get_log_rate_analysis_type'; export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; -export { type LogRateHistogramItem } from './log_rate_histogram_item'; -export { type DocumentCountStatsChangePoint } from './types'; -export { - getSnappedWindowParameters, - getWindowParameters, - type WindowParameters, -} from './window_parameters'; +export type { LogRateHistogramItem } from './log_rate_histogram_item'; +export type { DocumentCountStatsChangePoint } from './types'; +export type { WindowParameters } from './log_rate_analysis/window_parameters'; +export { getSnappedWindowParameters } from './log_rate_analysis/get_snapped_window_parameters'; +export { getWindowParameters } from './log_rate_analysis/get_window_parameters'; +export { getWindowParametersForTrigger } from './log_rate_analysis/get_window_parameters_for_trigger'; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.ts new file mode 100644 index 0000000000000..7421b06e5ab44 --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { WindowParameters } from './window_parameters'; + +/** + * + * Converts window paramaters from the brushes to “snap” the brushes to the chart histogram bar width and ensure timestamps + * correspond to bucket timestamps + * + * @param windowParameters time range definition for baseline and deviation to be used by log rate analysis + * @param snapTimestamps time range definition that always corresponds to histogram bucket timestamps + * @returns WindowParameters + */ +export const getSnappedWindowParameters = ( + windowParameters: WindowParameters, + snapTimestamps: number[] +): WindowParameters => { + const snappedBaselineMin = snapTimestamps.reduce((pts, cts) => { + if ( + Math.abs(cts - windowParameters.baselineMin) < Math.abs(pts - windowParameters.baselineMin) + ) { + return cts; + } + return pts; + }, snapTimestamps[0]); + const baselineMaxTimestamps = snapTimestamps.filter((ts) => ts > snappedBaselineMin); + + const snappedBaselineMax = baselineMaxTimestamps.reduce((pts, cts) => { + if ( + Math.abs(cts - windowParameters.baselineMax) < Math.abs(pts - windowParameters.baselineMax) + ) { + return cts; + } + return pts; + }, baselineMaxTimestamps[0]); + const deviationMinTss = baselineMaxTimestamps.filter((ts) => ts > snappedBaselineMax); + + const snappedDeviationMin = deviationMinTss.reduce((pts, cts) => { + if ( + Math.abs(cts - windowParameters.deviationMin) < Math.abs(pts - windowParameters.deviationMin) + ) { + return cts; + } + return pts; + }, deviationMinTss[0]); + const deviationMaxTss = deviationMinTss.filter((ts) => ts > snappedDeviationMin); + + const snappedDeviationMax = deviationMaxTss.reduce((pts, cts) => { + if ( + Math.abs(cts - windowParameters.deviationMax) < Math.abs(pts - windowParameters.deviationMax) + ) { + return cts; + } + return pts; + }, deviationMaxTss[0]); + + return { + baselineMin: snappedBaselineMin, + baselineMax: snappedBaselineMax, + deviationMin: snappedDeviationMin, + deviationMax: snappedDeviationMax, + }; +}; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts new file mode 100644 index 0000000000000..d1cd818b6d4fc --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { WindowParameters } from './window_parameters'; + +/** + * Given a point in time (e.g. where a user clicks), use simple heuristics to compute: + * + * 1. The time window around the click to evaluate for changes + * 2. The historical time window prior to the click to use as a baseline. + * + * The philosophy here is that charts are displayed with different granularities according to their + * overall time window. We select the log deviation and historical time windows inline with the + * overall time window. + * + * The algorithm for doing this is based on the typical granularities that exist in machine data. + * + * @param clickTime timestamp of the clicked log rate deviation. + * @param minTime minimum timestamp of the time window to be analysed + * @param maxTime maximum timestamp of the time window to be analysed + * @returns WindowParameters + */ +export const getWindowParameters = ( + clickTime: number, + minTime: number, + maxTime: number, + clickTimeUpper?: number, + windowGapOverride?: number +): WindowParameters => { + const totalWindow = maxTime - minTime; + + // min deviation window + const minDeviationWindow = 10 * 60 * 1000; // 10min + const minBaselineWindow = 30 * 60 * 1000; // 30min + const minWindowGap = 5 * 60 * 1000; // 5min + + // work out bounds as done in the original notebooks, + // with the deviation window aiming to be a 1/10 + // of the size of the total window and the baseline window + // being 3.5/10 of the total window. + const deviationWindow = Math.max(totalWindow / 10, minDeviationWindow); + const baselineWindow = Math.max(totalWindow / 3.5, minBaselineWindow); + const windowGap = windowGapOverride ?? Math.max(totalWindow / 10, minWindowGap); + + const deviationMin = clickTimeUpper ? clickTime : clickTime - deviationWindow / 2; + const deviationMax = clickTimeUpper ? clickTimeUpper : clickTime + deviationWindow / 2; + + const baselineMax = deviationMin - windowGap; + const baselineMin = baselineMax - baselineWindow; + + return { + baselineMin: Math.round(baselineMin), + baselineMax: Math.round(baselineMax), + deviationMin: Math.round(deviationMin), + deviationMax: Math.round(deviationMax), + }; +}; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts new file mode 100644 index 0000000000000..c6a2c9fb45d2e --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DocumentCountStatsChangePoint } from '../types'; +import { getWindowParameters } from './get_window_parameters'; +import type { WindowParameters } from './window_parameters'; + +export function getWindowParametersForTrigger( + startRange: number | WindowParameters, + interval: number, + timeRangeEarliest: number, + timeRangeLatest: number, + changePoint?: DocumentCountStatsChangePoint +) { + if ( + typeof startRange === 'number' && + changePoint && + startRange >= changePoint.lower && + startRange <= changePoint.upper + ) { + return getWindowParameters( + changePoint.lower + interval, + timeRangeEarliest, + timeRangeLatest + interval, + changePoint.upper, + interval + ); + } else if (typeof startRange === 'number') { + return getWindowParameters( + startRange + interval / 2, + timeRangeEarliest, + timeRangeLatest + interval + ); + } + + return startRange; +} diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/window_parameters.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/window_parameters.ts new file mode 100644 index 0000000000000..e570655fc1136 --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/window_parameters.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; + +/** + * Time range definition for baseline and deviation to be used by log rate analysis. + * + * @export + * @interface WindowParameters + * @typedef {WindowParameters} + */ +export interface WindowParameters { + /** Baseline minimum value */ + baselineMin: number; + /** Baseline maximum value */ + baselineMax: number; + /** Deviation minimum value */ + deviationMin: number; + /** Deviation maximum value */ + deviationMax: number; +} + +/** + * Type guard for WindowParameters + * + * @param {unknown} arg - The argument to be checked. + * @returns {arg is WindowParameters} + */ +export const isWindowParameters = (arg: unknown): arg is WindowParameters => + isPopulatedObject(arg, ['baselineMin', 'baselineMax', 'deviationMin', 'deviationMax']) && + Object.values(arg).every((d) => typeof d === 'number'); diff --git a/x-pack/packages/ml/aiops_utils/window_parameters.ts b/x-pack/packages/ml/aiops_utils/window_parameters.ts deleted file mode 100644 index 26e608d8852e1..0000000000000 --- a/x-pack/packages/ml/aiops_utils/window_parameters.ts +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isPopulatedObject } from '@kbn/ml-is-populated-object'; - -/** - * Time range definition for baseline and deviation to be used by log rate analysis. - * - * @export - * @interface WindowParameters - * @typedef {WindowParameters} - */ -export interface WindowParameters { - /** Baseline minimum value */ - baselineMin: number; - /** Baseline maximum value */ - baselineMax: number; - /** Deviation minimum value */ - deviationMin: number; - /** Deviation maximum value */ - deviationMax: number; -} - -/** - * Type guard for WindowParameters - * - * @param {unknown} arg - The argument to be checked. - * @returns {arg is WindowParameters} - */ -export const isWindowParameters = (arg: unknown): arg is WindowParameters => - isPopulatedObject(arg, ['baselineMin', 'baselineMax', 'deviationMin', 'deviationMax']) && - Object.values(arg).every((d) => typeof d === 'number'); - -/** - * Given a point in time (e.g. where a user clicks), use simple heuristics to compute: - * - * 1. The time window around the click to evaluate for changes - * 2. The historical time window prior to the click to use as a baseline. - * - * The philosophy here is that charts are displayed with different granularities according to their - * overall time window. We select the log deviation and historical time windows inline with the - * overall time window. - * - * The algorithm for doing this is based on the typical granularities that exist in machine data. - * - * @param clickTime timestamp of the clicked log rate deviation. - * @param minTime minimum timestamp of the time window to be analysed - * @param maxTime maximum timestamp of the time window to be analysed - * @returns WindowParameters - */ -export const getWindowParameters = ( - clickTime: number, - minTime: number, - maxTime: number, - clickTimeUpper?: number, - windowGapOverride?: number -): WindowParameters => { - const totalWindow = maxTime - minTime; - - // min deviation window - const minDeviationWindow = 10 * 60 * 1000; // 10min - const minBaselineWindow = 30 * 60 * 1000; // 30min - const minWindowGap = 5 * 60 * 1000; // 5min - - // work out bounds as done in the original notebooks, - // with the deviation window aiming to be a 1/10 - // of the size of the total window and the baseline window - // being 3.5/10 of the total window. - const deviationWindow = Math.max(totalWindow / 10, minDeviationWindow); - const baselineWindow = Math.max(totalWindow / 3.5, minBaselineWindow); - const windowGap = windowGapOverride ?? Math.max(totalWindow / 10, minWindowGap); - - const deviationMin = clickTimeUpper ? clickTime : clickTime - deviationWindow / 2; - const deviationMax = clickTimeUpper ? clickTimeUpper : clickTime + deviationWindow / 2; - - const baselineMax = deviationMin - windowGap; - const baselineMin = baselineMax - baselineWindow; - - return { - baselineMin: Math.round(baselineMin), - baselineMax: Math.round(baselineMax), - deviationMin: Math.round(deviationMin), - deviationMax: Math.round(deviationMax), - }; -}; - -/** - * - * Converts window paramaters from the brushes to “snap” the brushes to the chart histogram bar width and ensure timestamps - * correspond to bucket timestamps - * - * @param windowParameters time range definition for baseline and deviation to be used by log rate analysis - * @param snapTimestamps time range definition that always corresponds to histogram bucket timestamps - * @returns WindowParameters - */ -export const getSnappedWindowParameters = ( - windowParameters: WindowParameters, - snapTimestamps: number[] -): WindowParameters => { - const snappedBaselineMin = snapTimestamps.reduce((pts, cts) => { - if ( - Math.abs(cts - windowParameters.baselineMin) < Math.abs(pts - windowParameters.baselineMin) - ) { - return cts; - } - return pts; - }, snapTimestamps[0]); - const baselineMaxTimestamps = snapTimestamps.filter((ts) => ts > snappedBaselineMin); - - const snappedBaselineMax = baselineMaxTimestamps.reduce((pts, cts) => { - if ( - Math.abs(cts - windowParameters.baselineMax) < Math.abs(pts - windowParameters.baselineMax) - ) { - return cts; - } - return pts; - }, baselineMaxTimestamps[0]); - const deviationMinTss = baselineMaxTimestamps.filter((ts) => ts > snappedBaselineMax); - - const snappedDeviationMin = deviationMinTss.reduce((pts, cts) => { - if ( - Math.abs(cts - windowParameters.deviationMin) < Math.abs(pts - windowParameters.deviationMin) - ) { - return cts; - } - return pts; - }, deviationMinTss[0]); - const deviationMaxTss = deviationMinTss.filter((ts) => ts > snappedDeviationMin); - - const snappedDeviationMax = deviationMaxTss.reduce((pts, cts) => { - if ( - Math.abs(cts - windowParameters.deviationMax) < Math.abs(pts - windowParameters.deviationMax) - ) { - return cts; - } - return pts; - }, deviationMaxTss[0]); - - return { - baselineMin: snappedBaselineMin, - baselineMax: snappedBaselineMax, - deviationMin: snappedDeviationMin, - deviationMax: snappedDeviationMax, - }; -}; From 54607853d398a8837e0e5fa709bff95fe784b0c4 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Mar 2024 13:48:14 +0100 Subject: [PATCH 04/17] LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR --- x-pack/packages/ml/aiops_utils/index.ts | 1 + .../ml/aiops_utils/log_rate_analysis/constants.ts | 8 ++++++++ .../log_rate_analysis_content.tsx | 3 ++- .../public/components/mini_histogram/mini_histogram.tsx | 7 +++++-- 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts diff --git a/x-pack/packages/ml/aiops_utils/index.ts b/x-pack/packages/ml/aiops_utils/index.ts index 70b33c6d5fa45..8b168ba3b56a5 100644 --- a/x-pack/packages/ml/aiops_utils/index.ts +++ b/x-pack/packages/ml/aiops_utils/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +export { LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR } from './log_rate_analysis/constants'; export { getLogRateAnalysisType } from './get_log_rate_analysis_type'; export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; export type { LogRateHistogramItem } from './log_rate_histogram_item'; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts new file mode 100644 index 0000000000000..205cc63dabdf7 --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR = 'orange'; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 28ddd69e5e905..329e9bb4fc394 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -17,6 +17,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { DataView } from '@kbn/data-views-plugin/public'; import { + LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType, type WindowParameters, @@ -185,7 +186,7 @@ export const LogRateAnalysisContent: FC = ({ const barStyle = { rect: { opacity: 1, - fill: 'orange', + fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, }, }; const barStyleAccessor: BarStyleAccessor | undefined = documentCountStats?.changePoint diff --git a/x-pack/plugins/aiops/public/components/mini_histogram/mini_histogram.tsx b/x-pack/plugins/aiops/public/components/mini_histogram/mini_histogram.tsx index 5049cea733655..53bfb48a93f5b 100644 --- a/x-pack/plugins/aiops/public/components/mini_histogram/mini_histogram.tsx +++ b/x-pack/plugins/aiops/public/components/mini_histogram/mini_histogram.tsx @@ -13,10 +13,11 @@ import type { PartialTheme } from '@elastic/charts'; import { Chart, BarSeries, ScaleType, Settings, Tooltip, TooltipType } from '@elastic/charts'; import { EuiLoadingChart, EuiTextColor } from '@elastic/eui'; +import { LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR } from '@kbn/aiops-utils'; import { FormattedMessage } from '@kbn/i18n-react'; import type { SignificantItemHistogramItem } from '@kbn/ml-agg-utils'; - import { i18n } from '@kbn/i18n'; + import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { useEuiTheme } from '../../hooks/use_eui_theme'; @@ -94,7 +95,9 @@ export const MiniHistogram: FC = ({ } const barColor = barColorOverride ? [barColorOverride] : undefined; - const barHighlightColor = barHighlightColorOverride ? [barHighlightColorOverride] : ['orange']; + const barHighlightColor = barHighlightColorOverride + ? [barHighlightColorOverride] + : [LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR]; return (
From 60a3330bf4241a3dfbb20c34638c26cdfd767b57 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Mar 2024 14:07:13 +0100 Subject: [PATCH 05/17] getExtendedChangePoint --- x-pack/packages/ml/aiops_utils/index.ts | 12 ++++--- .../get_extended_change_point.ts | 35 +++++++++++++++++++ .../get_log_rate_analysis_type.test.ts | 0 .../get_log_rate_analysis_type.ts | 2 +- .../get_window_parameters_for_trigger.ts | 10 +++--- .../log_rate_analysis_type.ts | 0 .../log_rate_histogram_item.ts | 0 .../{ => log_rate_analysis}/types.ts | 4 +-- .../log_rate_analysis_content.tsx | 4 +-- .../aiops/public/get_document_stats.ts | 29 ++------------- 10 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts rename x-pack/packages/ml/aiops_utils/{ => log_rate_analysis}/get_log_rate_analysis_type.test.ts (100%) rename x-pack/packages/ml/aiops_utils/{ => log_rate_analysis}/get_log_rate_analysis_type.ts (94%) rename x-pack/packages/ml/aiops_utils/{ => log_rate_analysis}/log_rate_analysis_type.ts (100%) rename x-pack/packages/ml/aiops_utils/{ => log_rate_analysis}/log_rate_histogram_item.ts (100%) rename x-pack/packages/ml/aiops_utils/{ => log_rate_analysis}/types.ts (97%) diff --git a/x-pack/packages/ml/aiops_utils/index.ts b/x-pack/packages/ml/aiops_utils/index.ts index 8b168ba3b56a5..e2469cb4d5cbe 100644 --- a/x-pack/packages/ml/aiops_utils/index.ts +++ b/x-pack/packages/ml/aiops_utils/index.ts @@ -6,11 +6,15 @@ */ export { LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR } from './log_rate_analysis/constants'; -export { getLogRateAnalysisType } from './get_log_rate_analysis_type'; -export { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; -export type { LogRateHistogramItem } from './log_rate_histogram_item'; -export type { DocumentCountStatsChangePoint } from './types'; +export { getLogRateAnalysisType } from './log_rate_analysis/get_log_rate_analysis_type'; +export { + LOG_RATE_ANALYSIS_TYPE, + type LogRateAnalysisType, +} from './log_rate_analysis/log_rate_analysis_type'; +export type { LogRateHistogramItem } from './log_rate_analysis/log_rate_histogram_item'; +export type { DocumentCountStatsChangePoint } from './log_rate_analysis/types'; export type { WindowParameters } from './log_rate_analysis/window_parameters'; export { getSnappedWindowParameters } from './log_rate_analysis/get_snapped_window_parameters'; export { getWindowParameters } from './log_rate_analysis/get_window_parameters'; export { getWindowParametersForTrigger } from './log_rate_analysis/get_window_parameters_for_trigger'; +export { getExtendedChangePoint } from './log_rate_analysis/get_extended_change_point'; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts new file mode 100644 index 0000000000000..e9ccc9eab34df --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mean } from 'd3-array'; + +export const getExtendedChangePoint = (buckets: Record, changePointTs: number) => { + const bucketKeys = Object.keys(buckets); + const bucketValues = Object.values(buckets); + const meanValue = Math.round(mean(bucketValues) ?? 0); + const cpIndex = bucketKeys.findIndex((d) => +d === changePointTs); + const cpValue = buckets[changePointTs]; + + let lIndex = cpIndex - 1; + let uIndex = cpIndex + 1; + + while ( + lIndex >= 0 && + Math.abs(bucketValues[lIndex] - meanValue) > Math.abs(bucketValues[lIndex] - cpValue) + ) { + lIndex--; + } + + while ( + uIndex < bucketValues.length && + Math.abs(bucketValues[uIndex] - meanValue) > Math.abs(bucketValues[uIndex] - cpValue) + ) { + uIndex++; + } + + return { startTs: +bucketKeys[lIndex], endTs: +bucketKeys[uIndex] }; +}; diff --git a/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.test.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_log_rate_analysis_type.test.ts similarity index 100% rename from x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.test.ts rename to x-pack/packages/ml/aiops_utils/log_rate_analysis/get_log_rate_analysis_type.test.ts diff --git a/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_log_rate_analysis_type.ts similarity index 94% rename from x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts rename to x-pack/packages/ml/aiops_utils/log_rate_analysis/get_log_rate_analysis_type.ts index dc17d720bc42c..0303d903da05b 100644 --- a/x-pack/packages/ml/aiops_utils/get_log_rate_analysis_type.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_log_rate_analysis_type.ts @@ -9,7 +9,7 @@ import { median } from 'd3-array'; import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from './log_rate_analysis_type'; import type { LogRateHistogramItem } from './log_rate_histogram_item'; -import type { WindowParameters } from './log_rate_analysis/window_parameters'; +import type { WindowParameters } from './window_parameters'; /** * Identify the log rate analysis type based on the baseline/deviation diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts index c6a2c9fb45d2e..3f0b021789b82 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { DocumentCountStatsChangePoint } from '../types'; +import type { DocumentCountStatsChangePoint } from './types'; import { getWindowParameters } from './get_window_parameters'; import type { WindowParameters } from './window_parameters'; @@ -19,14 +19,14 @@ export function getWindowParametersForTrigger( if ( typeof startRange === 'number' && changePoint && - startRange >= changePoint.lower && - startRange <= changePoint.upper + startRange >= changePoint.startTs && + startRange <= changePoint.endTs ) { return getWindowParameters( - changePoint.lower + interval, + changePoint.startTs + interval, timeRangeEarliest, timeRangeLatest + interval, - changePoint.upper, + changePoint.endTs, interval ); } else if (typeof startRange === 'number') { diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis_type.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/log_rate_analysis_type.ts similarity index 100% rename from x-pack/packages/ml/aiops_utils/log_rate_analysis_type.ts rename to x-pack/packages/ml/aiops_utils/log_rate_analysis/log_rate_analysis_type.ts diff --git a/x-pack/packages/ml/aiops_utils/log_rate_histogram_item.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/log_rate_histogram_item.ts similarity index 100% rename from x-pack/packages/ml/aiops_utils/log_rate_histogram_item.ts rename to x-pack/packages/ml/aiops_utils/log_rate_analysis/log_rate_histogram_item.ts diff --git a/x-pack/packages/ml/aiops_utils/types.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts similarity index 97% rename from x-pack/packages/ml/aiops_utils/types.ts rename to x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts index f29869c3b4f73..9dca3df02fdfb 100644 --- a/x-pack/packages/ml/aiops_utils/types.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts @@ -47,7 +47,7 @@ export interface SimpleHierarchicalTreeNode { export interface DocumentCountStatsChangePoint { key: number; - lower: number; - upper: number; + startTs: number; + endTs: number; type: string; } diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 329e9bb4fc394..1682633fa6975 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -193,8 +193,8 @@ export const LogRateAnalysisContent: FC = ({ ? (d, g) => { return g.specId === 'document_count' && documentCountStats?.changePoint && - d.x > documentCountStats.changePoint.lower && - d.x < documentCountStats.changePoint.upper + d.x > documentCountStats.changePoint.startTs && + d.x < documentCountStats.changePoint.endTs ? barStyle : null; } diff --git a/x-pack/plugins/aiops/public/get_document_stats.ts b/x-pack/plugins/aiops/public/get_document_stats.ts index 8e1417508f968..168219bf3340c 100644 --- a/x-pack/plugins/aiops/public/get_document_stats.ts +++ b/x-pack/plugins/aiops/public/get_document_stats.ts @@ -6,12 +6,11 @@ */ import { get } from 'lodash'; -import { mean } from 'd3-array'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import dateMath from '@kbn/datemath'; -import type { DocumentCountStatsChangePoint } from '@kbn/aiops-utils'; +import { getExtendedChangePoint, type DocumentCountStatsChangePoint } from '@kbn/aiops-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import type { SignificantItem } from '@kbn/ml-agg-utils'; import type { Query } from '@kbn/es-query'; @@ -183,29 +182,6 @@ export const processDocumentCountStats = ( return acc; }, {}); - const bucketKeys = Object.keys(buckets); - const bucketValues = Object.values(buckets); - const meanValue = Math.round(mean(bucketValues) ?? 0); - const cpIndex = bucketKeys.findIndex((d) => +d === changePointBase?.key); - const cpValue = changePointBase ? buckets[changePointBase.key] : 0; - - let lIndex = cpIndex - 1; - let uIndex = cpIndex + 1; - - while ( - lIndex >= 0 && - Math.abs(bucketValues[lIndex] - meanValue) > Math.abs(bucketValues[lIndex] - cpValue) - ) { - lIndex--; - } - - while ( - uIndex < bucketValues.length && - Math.abs(bucketValues[uIndex] - meanValue) > Math.abs(bucketValues[uIndex] - cpValue) - ) { - uIndex++; - } - const lastDocTimeStamp: string = Object.values(body.hits.hits[0]?.fields ?? [[]])[0][0]; const lastDocTimeStampMs = lastDocTimeStamp === undefined ? undefined : dateMath.parse(lastDocTimeStamp)?.valueOf(); @@ -221,8 +197,7 @@ export const processDocumentCountStats = ( ? { changePoint: { ...changePointBase, - lower: +bucketKeys[lIndex], - upper: +bucketKeys[uIndex], + ...getExtendedChangePoint(buckets, changePointBase?.key), }, } : {}), From f86352f1ffc7df4aea088211ba18191d1bb6448a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Mar 2024 14:26:44 +0100 Subject: [PATCH 06/17] jest unit tests --- .../__mocks__/date_histogram.ts | 84 +++++++++++++++++++ .../get_extended_change_point.test.ts | 19 +++++ .../get_snapped_window_parameters.test.ts | 36 ++++++++ 3 files changed, 139 insertions(+) create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/__mocks__/date_histogram.ts create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.test.ts create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.test.ts diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/__mocks__/date_histogram.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/__mocks__/date_histogram.ts new file mode 100644 index 0000000000000..f5eed9226bcca --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/__mocks__/date_histogram.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getDateHistogramBuckets = (): Record => ({ + 1654566600000: 4929, + 1654566900000: 4686, + 1654567200000: 5243, + 1654567500000: 4186, + 1654567800000: 5529, + 1654568100000: 6071, + 1654568400000: 4500, + 1654568700000: 6157, + 1654569000000: 4886, + 1654569300000: 5886, + 1654569600000: 4843, + 1654569900000: 4871, + 1654570200000: 5129, + 1654570500000: 4529, + 1654570800000: 5171, + 1654571100000: 6357, + 1654571400000: 4100, + 1654571700000: 4714, + 1654572000000: 5029, + 1654572300000: 4100, + 1654572600000: 5057, + 1654572900000: 5129, + 1654573200000: 4871, + 1654573500000: 4914, + 1654573800000: 4586, + 1654574100000: 3857, + 1654574400000: 3886, + 1654574700000: 5286, + 1654575000000: 4543, + 1654575300000: 5800, + 1654575600000: 4943, + 1654575900000: 5071, + 1654576200000: 6486, + 1654576500000: 5914, + 1654576800000: 5643, + 1654577100000: 6500, + 1654577400000: 7014, + 1654577700000: 5300, + 1654578000000: 6086, + 1654578300000: 5829, + 1654578600000: 6743, + 1654578900000: 7457, + 1654579200000: 5729, + 1654579500000: 6871, + 1654579800000: 7457, + 1654580100000: 6657, + 1654580400000: 8543, + 1654580700000: 8629, + 1654581000000: 8586, + 1654581300000: 7043, + 1654581600000: 8071, + 1654581900000: 8471, + 1654582200000: 12243, + 1654582500000: 10171, + 1654582800000: 10143, + 1654583100000: 11529, + 1654583400000: 10986, + 1654583700000: 10757, + 1654584000000: 12614, + 1654584300000: 11771, + 1654584600000: 11771, + 1654584900000: 11543, + 1654585200000: 10671, + 1654585500000: 14914, + 1654585800000: 12500, + 1654586100000: 15029, + 1654586400000: 99900, + 1654586700000: 78971, + 1654587000000: 20600, + 1654587300000: 4300, + 1654587600000: 11671, + 1654587900000: 2629, + 1654588200000: 2200, + 1654588500000: 13157, + 1654588800000: 2714, +}); diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.test.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.test.ts new file mode 100644 index 0000000000000..2da39a97cdecc --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDateHistogramBuckets } from './__mocks__/date_histogram'; +import { getExtendedChangePoint } from './get_extended_change_point'; + +describe('getExtendedChangePoint', () => { + test('returns the extended change point', () => { + const changePointTs = 1654586400000; + expect(getExtendedChangePoint(getDateHistogramBuckets(), changePointTs)).toEqual({ + endTs: 1654587000000, + startTs: 1654586100000, + }); + }); +}); diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.test.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.test.ts new file mode 100644 index 0000000000000..1556b32812af1 --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_window_parameters.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDateHistogramBuckets } from './__mocks__/date_histogram'; +import { getSnappedWindowParameters } from './get_snapped_window_parameters'; + +const windowParameters = { + baselineMin: 1654579807500, + baselineMax: 1654586107500, + deviationMin: 1654586400000, + deviationMax: 1654587007500, +}; + +const snapTimestamps = Object.keys(getDateHistogramBuckets()).map((d) => +d); + +describe('getSnappedWindowParameters', () => { + test('returns the snapped window parameters', () => { + const snappedWindowParameters = getSnappedWindowParameters(windowParameters, snapTimestamps); + + expect(getSnappedWindowParameters(windowParameters, snapTimestamps)).toEqual({ + baselineMax: 1654586100000, + baselineMin: 1654579800000, + deviationMax: 1654587000000, + deviationMin: 1654586400000, + }); + + expect(snapTimestamps.includes(snappedWindowParameters.baselineMin)).toBe(true); + expect(snapTimestamps.includes(snappedWindowParameters.baselineMax)).toBe(true); + expect(snapTimestamps.includes(snappedWindowParameters.deviationMin)).toBe(true); + expect(snapTimestamps.includes(snappedWindowParameters.deviationMax)).toBe(true); + }); +}); From 50a861f316dd443422b56e7e37fad594364459a2 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 19 Mar 2024 15:46:26 +0100 Subject: [PATCH 07/17] fix imports --- .../alert_details_app_section/components/log_rate_analysis.tsx | 2 +- .../components/alert_details_app_section/log_rate_analysis.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx b/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx index de7bc44abf8b1..f704af1d96b7f 100644 --- a/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx +++ b/x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx @@ -15,7 +15,7 @@ import { DataView } from '@kbn/data-views-plugin/common'; import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType, -} from '@kbn/aiops-utils/log_rate_analysis_type'; +} from '@kbn/aiops-utils/log_rate_analysis/log_rate_analysis_type'; import { LogRateAnalysisContent, type LogRateAnalysisResultsData } from '@kbn/aiops-plugin/public'; import { Rule } from '@kbn/alerting-plugin/common'; import { TopAlert } from '@kbn/observability-plugin/public'; diff --git a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx index 3ab5d1face9aa..879c188297b59 100644 --- a/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx +++ b/x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx @@ -12,7 +12,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType, -} from '@kbn/aiops-utils/log_rate_analysis_type'; +} from '@kbn/aiops-utils/log_rate_analysis/log_rate_analysis_type'; import { LogRateAnalysisContent, type LogRateAnalysisResultsData } from '@kbn/aiops-plugin/public'; import { Rule } from '@kbn/alerting-plugin/common'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; From 226d2e143d4d9dcd8b545abf0676b7bdd0168e10 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Mar 2024 10:18:52 +0100 Subject: [PATCH 08/17] fix imports --- .../artificial_logs/filtered_frequent_item_sets.ts | 2 +- .../aiops_test_utils/artificial_logs/frequent_item_sets.ts | 2 +- .../log_rate_analysis/queries/fetch_frequent_item_sets.ts | 2 +- .../queries/fetch_terms_2_categories_counts.ts | 5 ++++- .../log_rate_analysis/queries/get_field_value_pair_counts.ts | 2 +- .../queries/get_groups_with_readded_duplicates.ts | 2 +- .../log_rate_analysis/queries/get_marked_duplicates.ts | 2 +- .../log_rate_analysis/queries/get_significant_item_groups.ts | 2 +- .../queries/get_simple_hierarchical_tree.ts | 2 +- .../queries/get_simple_hierarchical_tree_leaves.ts | 2 +- .../routes/log_rate_analysis/queries/get_value_counts.ts | 2 +- .../log_rate_analysis/queries/get_values_descending.ts | 2 +- .../queries/transform_significant_item_to_group.ts | 2 +- 13 files changed, 16 insertions(+), 13 deletions(-) diff --git a/x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts b/x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts index 1a665cf32cf4a..02dc1f059c90b 100644 --- a/x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts +++ b/x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '@kbn/aiops-utils/types'; +import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types'; export const filteredFrequentItemSets: ItemSet[] = [ { diff --git a/x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts b/x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts index 5c5a8fa0b79c9..8e432296350ba 100644 --- a/x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts +++ b/x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '@kbn/aiops-utils/types'; +import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types'; export const frequentItemSets: ItemSet[] = [ { diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts index aea40b0129fcf..de4bbfd98d44a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts @@ -18,7 +18,7 @@ import type { SignificantItemDuplicateGroup, ItemSet, FetchFrequentItemSetsResponse, -} from '@kbn/aiops-utils/types'; +} from '@kbn/aiops-utils/log_rate_analysis/types'; import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts index 998569f068d7f..f2d9699b343e6 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts @@ -12,7 +12,10 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { Logger } from '@kbn/logging'; import type { FieldValuePair, SignificantItem } from '@kbn/ml-agg-utils'; -import type { FetchFrequentItemSetsResponse, ItemSet } from '@kbn/aiops-utils/types'; +import type { + FetchFrequentItemSetsResponse, + ItemSet, +} from '@kbn/aiops-utils/log_rate_analysis/types'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts index e11be8ed2f2f0..8a7f233074b77 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts @@ -6,7 +6,7 @@ */ import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; -import type { FieldValuePairCounts } from '@kbn/aiops-utils/types'; +import type { FieldValuePairCounts } from '@kbn/aiops-utils/log_rate_analysis/types'; /** * Get a nested record of field/value pairs with counts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts index bfca0c3b82c58..c8fe282aef731 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts @@ -8,7 +8,7 @@ import { uniqWith, isEqual } from 'lodash'; import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; -import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/types'; +import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/log_rate_analysis/types'; export function getGroupsWithReaddedDuplicates( groups: SignificantItemGroup[], diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts index f0513a9323a76..ce3df3a89a0ef 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts @@ -6,7 +6,7 @@ */ import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; -import type { FieldValuePairCounts } from '@kbn/aiops-utils/types'; +import type { FieldValuePairCounts } from '@kbn/aiops-utils/log_rate_analysis/types'; /** * Analyse duplicate field/value pairs in groups. diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts index 1e0e140bb48d5..1bd0a70203fe2 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts @@ -8,7 +8,7 @@ import { uniqBy } from 'lodash'; import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils'; -import type { ItemSet } from '@kbn/aiops-utils/types'; +import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types'; import { duplicateIdentifier } from './duplicate_identifier'; import { groupDuplicates } from './fetch_frequent_item_sets'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts index 79722d67c8f6a..761d38d013593 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts @@ -6,7 +6,7 @@ */ import type { SignificantItem } from '@kbn/ml-agg-utils'; -import type { ItemSet, SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/types'; +import type { ItemSet, SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/log_rate_analysis/types'; import { getValueCounts } from './get_value_counts'; import { getValuesDescending } from './get_values_descending'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts index 27aba8ee54e00..367bd94f61e6e 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts @@ -8,7 +8,7 @@ import { orderBy } from 'lodash'; import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; import { stringHash } from '@kbn/ml-string-hash'; -import type { SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/types'; +import type { SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/log_rate_analysis/types'; /** * Get leaves from hierarchical tree. diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts index 624cfd7417689..1fa2bbeb9ce83 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '@kbn/aiops-utils/types'; +import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types'; export function getValueCounts(df: ItemSet[], field: string) { return df.reduce>((p, c) => { diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts index cc4f19ce388a4..60f1fb2b6a223 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '@kbn/aiops-utils/types'; +import type { ItemSet } from '@kbn/aiops-utils/log_rate_analysis/types'; import { getValueCounts } from './get_value_counts'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.ts index 88cb81dc3140a..bf8cbb2487c90 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.ts @@ -7,7 +7,7 @@ import { stringHash } from '@kbn/ml-string-hash'; import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils'; -import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/types'; +import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/log_rate_analysis/types'; export function transformSignificantItemToGroup( significantItem: SignificantItem, From 2de7f71c0b5f1c6dafcfe0eae5df208b93e139c9 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 20 Mar 2024 18:47:59 +0100 Subject: [PATCH 09/17] fix functional tests --- .../get_table_item_as_kql.ts | 7 ++- .../kibana_logs_data_view_test_data.ts | 58 ++++++++++++------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.ts b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.ts index 4374da0cec276..97717d43cb121 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.ts +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.ts @@ -5,19 +5,20 @@ * 2.0. */ -import { escapeKuery } from '@kbn/es-query'; +import { escapeKuery, escapeQuotes } from '@kbn/es-query'; import { isSignificantItem, type SignificantItem } from '@kbn/ml-agg-utils'; import type { GroupTableItem } from './types'; export const getTableItemAsKQL = (tableItem: GroupTableItem | SignificantItem) => { if (isSignificantItem(tableItem)) { - return `${escapeKuery(tableItem.fieldName)}:${escapeKuery(String(tableItem.fieldValue))}`; + return `${escapeKuery(tableItem.fieldName)}:"${escapeQuotes(String(tableItem.fieldValue))}"`; } return [ ...tableItem.groupItemsSortedByUniqueness.map( - ({ fieldName, fieldValue }) => `${escapeKuery(fieldName)}:${escapeKuery(String(fieldValue))}` + ({ fieldName, fieldValue }) => + `${escapeKuery(fieldName)}:"${escapeQuotes(String(fieldValue))}"` ), ].join(' AND '); }; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts index 1bf7f87aef5c9..1c1e534cdcb09 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts @@ -21,10 +21,10 @@ export const kibanaLogsDataViewTestData: TestData = { fieldSelectorApplyAvailable: true, action: { type: 'LogPatternAnalysis', - tableRowId: '1064853178', + tableRowId: '822370508', expected: { queryBar: - 'clientip:30.156.16.164 AND host.keyword:elastic-elastic-elastic.org AND ip:30.156.16.163 AND response.keyword:404 AND machine.os.keyword:win xp AND geo.dest:IN AND geo.srcdest:US\\:IN', + 'clientip:"30.156.16.164" AND geo.dest:"IN" AND geo.srcdest:"US:IN" AND host.keyword:"elastic-elastic-elastic.org" AND response.keyword:"404" AND ip:"30.156.16.163" AND machine.os.keyword:"win xp" AND agent.keyword:"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24" AND tags.keyword:"info" AND extension.keyword:""', totalDocCount: 100, }, }, @@ -41,87 +41,104 @@ export const kibanaLogsDataViewTestData: TestData = { searchQueryLanguage: 'kuery', searchString: '', wp: { - bMax: 1684368000000, - bMin: 1682899200000, - dMax: 1685491200000, - dMin: 1684886400000, + bMax: 1685059200000, + bMin: 1683590400000, + dMax: 1685232000000, + dMin: 1685145600000, }, }, }, analysisGroupsTable: [ { group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* referer: http://www.elastic-elastic-elastic.com/success/timothy-l-kopra* response.keyword: 404Showing 5 out of 8 items. 8 items unique to this group.', + '* clientip: 30.156.16.164* geo.dest: IN* geo.srcdest: US:IN* host.keyword: elastic-elastic-elastic.org* response.keyword: 404Showing 5 out of 11 items. 11 items unique to this group.', docCount: '100', }, ], filteredAnalysisGroupsTable: [ { group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* response.keyword: 404* machine.os.keyword: win xpShowing 5 out of 7 items. 7 items unique to this group.', + '* clientip: 30.156.16.164* geo.dest: IN* geo.srcdest: US:IN* host.keyword: elastic-elastic-elastic.org* response.keyword: 404Showing 5 out of 10 items. 10 items unique to this group.', docCount: '100', }, ], analysisTable: [ + { + fieldName: 'agent.keyword', + fieldValue: + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', + logRate: 'Chart type:bar chart', + pValue: '5.82e-12', + impact: 'High', + }, { fieldName: 'clientip', fieldValue: '30.156.16.164', logRate: 'Chart type:bar chart', - pValue: '3.10e-13', + pValue: '2.81e-53', + impact: 'High', + }, + { + fieldName: 'extension.keyword', + fieldValue: '', + logRate: 'Chart type:bar chart', + pValue: '5.72e-12', impact: 'High', }, { fieldName: 'geo.dest', fieldValue: 'IN', logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', + pValue: '8.35e-21', + impact: 'High', }, { fieldName: 'geo.srcdest', fieldValue: 'US:IN', logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', + pValue: '8.35e-21', + impact: 'High', }, { fieldName: 'host.keyword', fieldValue: 'elastic-elastic-elastic.org', logRate: 'Chart type:bar chart', - pValue: '7.14e-9', + pValue: '3.94e-45', impact: 'High', }, { fieldName: 'ip', fieldValue: '30.156.16.163', logRate: 'Chart type:bar chart', - pValue: '3.28e-13', + pValue: '9.50e-54', impact: 'High', }, { fieldName: 'machine.os.keyword', fieldValue: 'win xp', logRate: 'Chart type:bar chart', - pValue: '0.0000997', - impact: 'Medium', + pValue: '4.25e-18', + impact: 'High', }, { fieldName: 'referer', fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', logRate: 'Chart type:bar chart', - pValue: '4.74e-13', + pValue: '1.41e-53', impact: 'High', }, { fieldName: 'response.keyword', fieldValue: '404', logRate: 'Chart type:bar chart', - pValue: '0.00000604', - impact: 'Medium', + pValue: '2.10e-35', + impact: 'High', }, ], fieldSelectorPopover: [ + 'agent.keyword', 'clientip', + 'extension.keyword', 'geo.dest', 'geo.srcdest', 'host.keyword', @@ -129,6 +146,7 @@ export const kibanaLogsDataViewTestData: TestData = { 'machine.os.keyword', 'referer', 'response.keyword', + 'tags.keyword', ], }, }; From 6d43fdfcc0e414b92fb49aae2eb4368d5daa605d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 21 Mar 2024 07:59:03 +0100 Subject: [PATCH 10/17] jsdoc --- .../log_rate_analysis/constants.ts | 1 + .../get_extended_change_point.ts | 9 +++++++++ .../get_window_parameters.ts | 3 +++ .../get_window_parameters_for_trigger.ts | 19 ++++++++++++++++++- .../ml/aiops_utils/log_rate_analysis/types.ts | 8 ++++++++ 5 files changed, 39 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts index 205cc63dabdf7..0de3b6becc388 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/constants.ts @@ -5,4 +5,5 @@ * 2.0. */ +/** Highlighting color for charts */ export const LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR = 'orange'; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts index e9ccc9eab34df..c0e990dea705a 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_extended_change_point.ts @@ -7,6 +7,15 @@ import { mean } from 'd3-array'; +/** + * Calculates and returns an extended change point range based on the specified change point timestamp. + * + * @param buckets - An object where keys are bucket timestamps as strings + * and values are numeric values associated with each bucket. + * @param changePointTs - The timestamp of the change point as a number. This timestamp must + * be one of the keys in the `buckets` object. + * @returns An object containing two properties: `startTs` and `endTs`. + */ export const getExtendedChangePoint = (buckets: Record, changePointTs: number) => { const bucketKeys = Object.keys(buckets); const bucketValues = Object.values(buckets); diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts index d1cd818b6d4fc..2b8441b5e1c98 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters.ts @@ -22,6 +22,9 @@ import type { WindowParameters } from './window_parameters'; * @param clickTime timestamp of the clicked log rate deviation. * @param minTime minimum timestamp of the time window to be analysed * @param maxTime maximum timestamp of the time window to be analysed + * @param clickTimeUpper optional timestamp to treat clicktime and clickTimeUpper + * as a time range instead of point in time + * @param windowGapOverride optional override for the baseline/deviation gap * @returns WindowParameters */ export const getWindowParameters = ( diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts index 3f0b021789b82..1b75bdbdf0698 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_window_parameters_for_trigger.ts @@ -9,13 +9,30 @@ import type { DocumentCountStatsChangePoint } from './types'; import { getWindowParameters } from './get_window_parameters'; import type { WindowParameters } from './window_parameters'; +/** + * Calculates window parameters, adjusting the window based on a + * change point and interval. If a change point is specified and falls within + * the startRange, the window is adjusted around the change point. Otherwise, + * the window is determined by the startRange and interval. + * + * @param startRange The start timestamp or window parameters. If a number, + * it's the start timestamp; if an object, it's assumed to be + * window parameters and is returned directly. + * @param interval Interval in milliseconds for extending the window or + * adjusting the start range. + * @param timeRangeEarliest Earliest timestamp in milliseconds in the time range. + * @param timeRangeLatest Latest timestamp in milliseconds in the time range. + * @param changePoint Optional change point with `startTs` and `endTs` + * properties. Adjusts window parameters if within `startRange`. + * @returns Window parameters + */ export function getWindowParametersForTrigger( startRange: number | WindowParameters, interval: number, timeRangeEarliest: number, timeRangeLatest: number, changePoint?: DocumentCountStatsChangePoint -) { +): WindowParameters { if ( typeof startRange === 'number' && changePoint && diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts index 9dca3df02fdfb..a2b198f460a5b 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/types.ts @@ -45,9 +45,17 @@ export interface SimpleHierarchicalTreeNode { addNode: (node: SimpleHierarchicalTreeNode) => void; } +/** + * Represents a change point in document count statistics, + * identifying a significant change over time. + */ export interface DocumentCountStatsChangePoint { + /** Key is the timestamp of the change point. */ key: number; + /** The start timestamp of the change point period. */ startTs: number; + /** The end timestamp of the change point period. */ endTs: number; + /** The type of change point. */ type: string; } From 65ad5ed03fd8330bacc0278dd811dec26a559132 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 21 Mar 2024 09:59:16 +0100 Subject: [PATCH 11/17] fix jest test --- .../get_table_item_as_kql.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts index ff1a986ce4759..826f0ec66fa87 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts @@ -13,15 +13,15 @@ import { getTableItemAsKQL } from './get_table_item_as_kql'; describe('getTableItemAsKQL', () => { it('returns a KQL syntax for a significant item', () => { - expect(getTableItemAsKQL(significantTerms[0])).toBe('user:Peter'); - expect(getTableItemAsKQL(significantTerms[1])).toBe('response_code:500'); - expect(getTableItemAsKQL(significantTerms[2])).toBe('url:home.php'); - expect(getTableItemAsKQL(significantTerms[3])).toBe('url:login.php'); + expect(getTableItemAsKQL(significantTerms[0])).toBe('user:"Peter"'); + expect(getTableItemAsKQL(significantTerms[1])).toBe('response_code:"500"'); + expect(getTableItemAsKQL(significantTerms[2])).toBe('url:"home.php"'); + expect(getTableItemAsKQL(significantTerms[3])).toBe('url:"login.php"'); }); it('returns a KQL syntax for a group of significant items', () => { const groupTableItems = getGroupTableItems(finalSignificantItemGroups); - expect(getTableItemAsKQL(groupTableItems[0])).toBe('response_code:500 AND url:home.php'); - expect(getTableItemAsKQL(groupTableItems[1])).toBe('url:login.php AND response_code:500'); - expect(getTableItemAsKQL(groupTableItems[2])).toBe('user:Peter AND url:home.php'); + expect(getTableItemAsKQL(groupTableItems[0])).toBe('response_code:"500" AND url:"home.php"'); + expect(getTableItemAsKQL(groupTableItems[1])).toBe('url:"login.php" AND response_code:"500"'); + expect(getTableItemAsKQL(groupTableItems[2])).toBe('user:"Peter" AND url:"home.php"'); }); }); From a713d8b644bfd9b1d2e833ec5f1ffcb49d9cf076 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 21 Mar 2024 12:44:06 +0100 Subject: [PATCH 12/17] more jest tests --- .../significant_terms.ts | 60 ++++++++++++++ .../get_table_item_as_kql.test.ts | 45 ++++++++++- .../kibana_logs_data_view_test_data.ts | 79 ++----------------- 3 files changed, 110 insertions(+), 74 deletions(-) create mode 100644 x-pack/packages/ml/aiops_test_utils/kibana_sample_data_logs/significant_terms.ts diff --git a/x-pack/packages/ml/aiops_test_utils/kibana_sample_data_logs/significant_terms.ts b/x-pack/packages/ml/aiops_test_utils/kibana_sample_data_logs/significant_terms.ts new file mode 100644 index 0000000000000..890a6a7146399 --- /dev/null +++ b/x-pack/packages/ml/aiops_test_utils/kibana_sample_data_logs/significant_terms.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const kibanaSampleDataLogsSignificantTermsBase = [ + { + fieldName: 'agent.keyword', + fieldValue: + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', + pValue: '5.82e-12', + }, + { + fieldName: 'clientip', + fieldValue: '30.156.16.164', + pValue: '2.81e-53', + }, + { + fieldName: 'extension.keyword', + fieldValue: '', + pValue: '5.72e-12', + }, + { + fieldName: 'geo.dest', + fieldValue: 'IN', + pValue: '8.35e-21', + }, + { + fieldName: 'geo.srcdest', + fieldValue: 'US:IN', + pValue: '8.35e-21', + }, + { + fieldName: 'host.keyword', + fieldValue: 'elastic-elastic-elastic.org', + pValue: '3.94e-45', + }, + { + fieldName: 'ip', + fieldValue: '30.156.16.163', + pValue: '9.50e-54', + }, + { + fieldName: 'machine.os.keyword', + fieldValue: 'win xp', + pValue: '4.25e-18', + }, + { + fieldName: 'referer', + fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', + pValue: '1.41e-53', + }, + { + fieldName: 'response.keyword', + fieldValue: '404', + pValue: '2.10e-35', + }, +]; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts index 826f0ec66fa87..54291e22e03e8 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts @@ -7,21 +7,64 @@ import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups'; import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms'; +import { kibanaSampleDataLogsSignificantTermsBase } from '@kbn/aiops-test-utils/kibana_sample_data_logs/significant_terms'; +import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils'; import { getGroupTableItems } from './get_group_table_items'; import { getTableItemAsKQL } from './get_table_item_as_kql'; +const kibanaSampleDataLogsSignificantTerms: SignificantItem[] = + kibanaSampleDataLogsSignificantTermsBase.map((d) => ({ + ...d, + key: `${d.fieldName}:${d.fieldValue}`, + type: 'keyword', + doc_count: 1981, + bg_count: 553, + total_doc_count: 4669, + total_bg_count: 1975, + score: 47.38899434932384, + normalizedScore: 0.8328439168064725, + pValue: +d.pValue, + })); + +const kibanaSampleDataLogsGroups: SignificantItemGroup[] = [ + { + id: 'the-group-id', + group: kibanaSampleDataLogsSignificantTermsBase.map((d) => ({ + ...d, + key: `${d.fieldName}:${d.fieldValue}`, + type: 'keyword', + docCount: 1981, + pValue: +d.pValue, + })), + docCount: 792, + pValue: 0.00974308761016614, + }, +]; + describe('getTableItemAsKQL', () => { it('returns a KQL syntax for a significant item', () => { expect(getTableItemAsKQL(significantTerms[0])).toBe('user:"Peter"'); expect(getTableItemAsKQL(significantTerms[1])).toBe('response_code:"500"'); expect(getTableItemAsKQL(significantTerms[2])).toBe('url:"home.php"'); expect(getTableItemAsKQL(significantTerms[3])).toBe('url:"login.php"'); + + expect(getTableItemAsKQL(kibanaSampleDataLogsSignificantTerms[0])).toBe( + 'agent.keyword:"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"' + ); }); - it('returns a KQL syntax for a group of significant items', () => { + + it('returns a KQL syntax for a group of significant items for the artificial logs dataset', () => { const groupTableItems = getGroupTableItems(finalSignificantItemGroups); expect(getTableItemAsKQL(groupTableItems[0])).toBe('response_code:"500" AND url:"home.php"'); expect(getTableItemAsKQL(groupTableItems[1])).toBe('url:"login.php" AND response_code:"500"'); expect(getTableItemAsKQL(groupTableItems[2])).toBe('user:"Peter" AND url:"home.php"'); }); + + it('returns a KQL syntax for a group of significant items for the Kibana logs dataset', () => { + const groupTableItems = getGroupTableItems(kibanaSampleDataLogsGroups); + expect(getTableItemAsKQL(groupTableItems[0])).toBe( + 'agent.keyword:"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24" AND clientip:"30.156.16.164" AND extension.keyword:"" AND geo.dest:"IN" AND geo.srcdest:"US:IN" AND host.keyword:"elastic-elastic-elastic.org" AND ip:"30.156.16.163" AND machine.os.keyword:"win xp" AND referer:"http://www.elastic-elastic-elastic.com/success/timothy-l-kopra" AND response.keyword:"404"' + ); + }); }); diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts index 1c1e534cdcb09..f01ff18355f40 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts @@ -6,6 +6,7 @@ */ import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; +import { kibanaSampleDataLogsSignificantTermsBase } from '@kbn/aiops-test-utils/kibana_sample_data_logs/significant_terms'; import type { TestData } from '../../types'; @@ -62,79 +63,11 @@ export const kibanaLogsDataViewTestData: TestData = { docCount: '100', }, ], - analysisTable: [ - { - fieldName: 'agent.keyword', - fieldValue: - 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', - logRate: 'Chart type:bar chart', - pValue: '5.82e-12', - impact: 'High', - }, - { - fieldName: 'clientip', - fieldValue: '30.156.16.164', - logRate: 'Chart type:bar chart', - pValue: '2.81e-53', - impact: 'High', - }, - { - fieldName: 'extension.keyword', - fieldValue: '', - logRate: 'Chart type:bar chart', - pValue: '5.72e-12', - impact: 'High', - }, - { - fieldName: 'geo.dest', - fieldValue: 'IN', - logRate: 'Chart type:bar chart', - pValue: '8.35e-21', - impact: 'High', - }, - { - fieldName: 'geo.srcdest', - fieldValue: 'US:IN', - logRate: 'Chart type:bar chart', - pValue: '8.35e-21', - impact: 'High', - }, - { - fieldName: 'host.keyword', - fieldValue: 'elastic-elastic-elastic.org', - logRate: 'Chart type:bar chart', - pValue: '3.94e-45', - impact: 'High', - }, - { - fieldName: 'ip', - fieldValue: '30.156.16.163', - logRate: 'Chart type:bar chart', - pValue: '9.50e-54', - impact: 'High', - }, - { - fieldName: 'machine.os.keyword', - fieldValue: 'win xp', - logRate: 'Chart type:bar chart', - pValue: '4.25e-18', - impact: 'High', - }, - { - fieldName: 'referer', - fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', - logRate: 'Chart type:bar chart', - pValue: '1.41e-53', - impact: 'High', - }, - { - fieldName: 'response.keyword', - fieldValue: '404', - logRate: 'Chart type:bar chart', - pValue: '2.10e-35', - impact: 'High', - }, - ], + analysisTable: kibanaSampleDataLogsSignificantTermsBase.map((d) => ({ + ...d, + logRate: 'Chart type:bar chart', + impact: 'High', + })), fieldSelectorPopover: [ 'agent.keyword', 'clientip', From 8a15579d99a0ad5a047c024e1f432ebbb5d5ea5c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 25 Mar 2024 11:24:54 +0100 Subject: [PATCH 13/17] highlight auto-detected change point only in clear-brush state --- .../log_rate_analysis_content.tsx | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 1682633fa6975..557295db2fadd 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -189,16 +189,19 @@ export const LogRateAnalysisContent: FC = ({ fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, }, }; - const barStyleAccessor: BarStyleAccessor | undefined = documentCountStats?.changePoint - ? (d, g) => { - return g.specId === 'document_count' && - documentCountStats?.changePoint && - d.x > documentCountStats.changePoint.startTs && - d.x < documentCountStats.changePoint.endTs - ? barStyle - : null; - } - : undefined; + + // Used to highlight an auto-detected change point in the date histogram. + const barStyleAccessor: BarStyleAccessor | undefined = + isBrushCleared && documentCountStats?.changePoint + ? (d, g) => { + return g.specId === 'document_count' && + documentCountStats?.changePoint && + d.x > documentCountStats.changePoint.startTs && + d.x < documentCountStats.changePoint.endTs + ? barStyle + : null; + } + : undefined; return ( From 8392f8bc366c9846dc5c9f072a8b64f3104725aa Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 25 Mar 2024 16:53:41 +0100 Subject: [PATCH 14/17] update functional tests --- .../document_count_chart.tsx | 16 +--- x-pack/packages/ml/aiops_utils/index.ts | 1 + .../get_snapped_timestamps.ts | 22 +++++ .../log_rate_analysis_content.tsx | 93 ++++++++++++++++++- .../apps/aiops/log_rate_analysis.ts | 10 +- .../artificial_log_data_view_test_data.ts | 1 + .../farequote_data_view_test_data.ts | 1 + ...arequote_data_view_test_data_with_query.ts | 1 + .../kibana_logs_data_view_test_data.ts | 1 + x-pack/test/functional/apps/aiops/types.ts | 2 + .../services/aiops/log_rate_analysis_page.ts | 4 + 11 files changed, 136 insertions(+), 16 deletions(-) create mode 100644 x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index 1e3c412eca173..be97a44db8204 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -25,6 +25,7 @@ import { i18n } from '@kbn/i18n'; import type { IUiSettingsClient } from '@kbn/core/public'; import { getLogRateAnalysisType, + getSnappedTimestamps, getSnappedWindowParameters, getWindowParametersForTrigger, type DocumentCountStatsChangePoint, @@ -255,17 +256,10 @@ export const DocumentCountChart: FC = (props) => { // eslint-disable-next-line react-hooks/exhaustive-deps }, [chartPointsSplit, timeRangeEarliest, timeRangeLatest, interval]); - const snapTimestamps = useMemo(() => { - const timestamps: number[] = []; - let n = timeRangeEarliest; - - while (n <= timeRangeLatest + interval) { - timestamps.push(n); - n += interval; - } - - return timestamps; - }, [timeRangeEarliest, timeRangeLatest, interval]); + const snapTimestamps = useMemo( + () => getSnappedTimestamps(timeRangeEarliest, timeRangeLatest, interval), + [timeRangeEarliest, timeRangeLatest, interval] + ); const timefilterUpdateHandler = useCallback( (range: TimeFilterRange) => { diff --git a/x-pack/packages/ml/aiops_utils/index.ts b/x-pack/packages/ml/aiops_utils/index.ts index e2469cb4d5cbe..c94e362943d07 100644 --- a/x-pack/packages/ml/aiops_utils/index.ts +++ b/x-pack/packages/ml/aiops_utils/index.ts @@ -14,6 +14,7 @@ export { export type { LogRateHistogramItem } from './log_rate_analysis/log_rate_histogram_item'; export type { DocumentCountStatsChangePoint } from './log_rate_analysis/types'; export type { WindowParameters } from './log_rate_analysis/window_parameters'; +export { getSnappedTimestamps } from './log_rate_analysis/get_snapped_timestamps'; export { getSnappedWindowParameters } from './log_rate_analysis/get_snapped_window_parameters'; export { getWindowParameters } from './log_rate_analysis/get_window_parameters'; export { getWindowParametersForTrigger } from './log_rate_analysis/get_window_parameters_for_trigger'; diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts new file mode 100644 index 0000000000000..89489dd37cbf0 --- /dev/null +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getSnappedTimestamps = ( + timeRangeEarliest: number, + timeRangeLatest: number, + interval: number +) => { + const timestamps: number[] = []; + let n = timeRangeEarliest; + + while (n <= timeRangeLatest + interval) { + timestamps.push(n); + n += interval; + } + + return timestamps; +}; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 557295db2fadd..03a8271359214 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -6,8 +6,8 @@ */ import { isEqual } from 'lodash'; -import React, { useEffect, useMemo, useRef, useState, type FC } from 'react'; -import { EuiEmptyPrompt, EuiHorizontalRule, EuiPanel } from '@elastic/eui'; +import React, { useCallback, useEffect, useMemo, useRef, useState, type FC } from 'react'; +import { EuiButton, EuiEmptyPrompt, EuiHorizontalRule, EuiPanel } from '@elastic/eui'; import type { Moment } from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -17,6 +17,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { DataView } from '@kbn/data-views-plugin/public'; import { + getWindowParametersForTrigger, + getSnappedTimestamps, + getSnappedWindowParameters, LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType, @@ -203,6 +206,38 @@ export const LogRateAnalysisContent: FC = ({ } : undefined; + const triggerAnalysis = useCallback(() => { + if (documentCountStats) { + const { interval, timeRangeEarliest, timeRangeLatest, changePoint } = documentCountStats; + + if (changePoint && interval && timeRangeEarliest && timeRangeLatest) { + const wp = getWindowParametersForTrigger( + changePoint.startTs, + interval, + timeRangeEarliest, + timeRangeLatest, + changePoint + ); + + const snapTimestamps = getSnappedTimestamps(timeRangeEarliest, timeRangeLatest, interval); + + const wpSnap = getSnappedWindowParameters(wp, snapTimestamps); + + if (brushSelectionUpdate !== undefined) { + brushSelectionUpdate( + wpSnap, + true, + changePoint.type === LOG_RATE_ANALYSIS_TYPE.DIP + ? LOG_RATE_ANALYSIS_TYPE.DIP + : LOG_RATE_ANALYSIS_TYPE.SPIKE + ); + } + } + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [documentCountStats]); + return ( {documentCountStats !== undefined && ( @@ -242,7 +277,59 @@ export const LogRateAnalysisContent: FC = ({ embeddingOrigin={embeddingOrigin} /> )} - {windowParameters === undefined && ( + {windowParameters === undefined && documentCountStats?.changePoint && ( + + {documentCountStats?.changePoint.type === LOG_RATE_ANALYSIS_TYPE.SPIKE && ( + + )} + {documentCountStats?.changePoint.type === LOG_RATE_ANALYSIS_TYPE.DIP && ( + + )} + {documentCountStats?.changePoint.type !== LOG_RATE_ANALYSIS_TYPE.SPIKE && + documentCountStats?.changePoint.type !== LOG_RATE_ANALYSIS_TYPE.DIP && ( + + )} + + } + titleSize="xs" + body={ + <> +

+ +

+ + + + + } + data-test-subj="aiopsChangePointDetectedPrompt" + /> + )} + {windowParameters === undefined && documentCountStats?.changePoint === undefined && ( ; fieldSelectorPopover: string[]; + prompt: 'empty' | 'change-point'; } export interface TestData { diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts index 04b24ab5ae6e5..8db9e33229a7f 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts @@ -108,6 +108,10 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr await testSubjects.existOrFail(`aiopsSearchPanel`); }, + async assertChangePointDetectedPromptExists() { + await testSubjects.existOrFail(`aiopsChangePointDetectedPrompt`); + }, + async assertNoWindowParametersEmptyPromptExists() { await testSubjects.existOrFail(`aiopsNoWindowParametersEmptyPrompt`); }, From fb68fc8a722c2288f17c229428c154f0298225a0 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 26 Mar 2024 09:12:20 +0100 Subject: [PATCH 15/17] jsdoc --- .../log_rate_analysis/get_snapped_timestamps.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts index 89489dd37cbf0..f094f1bf44ed7 100644 --- a/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts +++ b/x-pack/packages/ml/aiops_utils/log_rate_analysis/get_snapped_timestamps.ts @@ -5,6 +5,14 @@ * 2.0. */ +/** + * Generates an array of timestamps evenly spaced within a given time range. + * + * @param timeRangeEarliest The earliest timestamp in the time range. + * @param timeRangeLatest The latest timestamp in the time range. + * @param interval The interval between timestamps in milliseconds. + * @returns Array of timestamps spaced by the specified interval within the given range. + */ export const getSnappedTimestamps = ( timeRangeEarliest: number, timeRangeLatest: number, From d8daf32caf8f0d39fec91d81c063f5989a8c800b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 26 Mar 2024 13:33:01 +0100 Subject: [PATCH 16/17] fix brushes when using run analysis button from prompt --- .../log_rate_analysis_content/log_rate_analysis_content.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index 03a8271359214..b91517ad3c098 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -224,6 +224,7 @@ export const LogRateAnalysisContent: FC = ({ const wpSnap = getSnappedWindowParameters(wp, snapTimestamps); if (brushSelectionUpdate !== undefined) { + setInitialAnalysisStart(wpSnap); brushSelectionUpdate( wpSnap, true, From d75483023aed5c98cfeb39322d8ca63f3e93e161 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 26 Mar 2024 18:11:54 +0100 Subject: [PATCH 17/17] fix run analysis button --- .../log_rate_analysis_content.tsx | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx index b91517ad3c098..551f45a175cb0 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_content/log_rate_analysis_content.tsx @@ -220,23 +220,11 @@ export const LogRateAnalysisContent: FC = ({ ); const snapTimestamps = getSnappedTimestamps(timeRangeEarliest, timeRangeLatest, interval); - const wpSnap = getSnappedWindowParameters(wp, snapTimestamps); - if (brushSelectionUpdate !== undefined) { - setInitialAnalysisStart(wpSnap); - brushSelectionUpdate( - wpSnap, - true, - changePoint.type === LOG_RATE_ANALYSIS_TYPE.DIP - ? LOG_RATE_ANALYSIS_TYPE.DIP - : LOG_RATE_ANALYSIS_TYPE.SPIKE - ); - } + setInitialAnalysisStart(wpSnap); } } - - // eslint-disable-next-line react-hooks/exhaustive-deps }, [documentCountStats]); return (