Sorting: add outlier detection as a sorting option (#658)

* chore(sorting): catch sorting exceptions

* feat(SortByScene): add outliers as an option

* feat(sorting): integrate outlier detection

* chore: move reporting outside of the reducer function

* chore: update description

* chore: check for wasm support before initializing augurs

* chore: update words

project-words.txt

@@ -466,4 +466,5 @@ inital
 lezer
 logql
 subqueries
+dbscan
 svgs

src/Components/ServiceScene/Breakdowns/SortByScene.tsx

@@ -28,6 +28,11 @@ export class SortByScene extends SceneObjectBase<SortBySceneState> {
           label: 'Most relevant',
           description: 'Smart ordering of graphs based on the most significant spikes in the data',
         },
+        {
+          value: 'outliers',
+          label: 'Detected outliers',
+          description: 'Order by the amount of detected outliers in the data',
+        },
         {
           value: ReducerID.stdDev,
           label: 'Widest spread',

src/module.ts

@@ -3,9 +3,11 @@ import { App } from 'Components/App';
 import init from '@bsull/augurs';
 import { linkConfigs } from 'services/extensions/links';
 import { init as initRuntimeDs } from 'services/datasource';
+import { wasmSupported } from 'services/sorting';
 
-// eslint-disable-next-line no-console
-init().then(() => console.debug('Grafana ML initialized'));
+if (wasmSupported()) {
+  init();
+}
 
 export const plugin = new AppPlugin<{}>().setRootPage(App);
 

src/services/sorting.ts

@@ -1,5 +1,5 @@
-import { ChangepointDetector } from '@bsull/augurs';
-import { DataFrame, FieldType, ReducerID, doStandardCalcs, fieldReducers } from '@grafana/data';
+import { ChangepointDetector, OutlierDetector, OutlierOutput } from '@bsull/augurs';
+import { DataFrame, FieldType, ReducerID, doStandardCalcs, fieldReducers, outerJoinDataFrames } from '@grafana/data';
 import { getLabelValueFromDataFrame } from './levels';
 import { memoize } from 'lodash';
 import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from './analytics';
@@ -10,18 +10,22 @@ export const sortSeries = memoize(
       return sortSeriesByName(series, direction);
     }
 
+    if (sortBy === 'outliers') {
+      initOutlierDetector(series);
+    }
+
     const reducer = (dataFrame: DataFrame) => {
-      if (sortBy === 'changepoint') {
-        if (wasmSupported()) {
+      // ML & Wasm sorting options
+      try {
+        if (sortBy === 'changepoint') {
           return calculateDataFrameChangepoints(dataFrame);
-        } else {
-          console.warn('Changepoint not supported, using stdDev');
-          reportAppInteraction(
-            USER_EVENTS_PAGES.service_details,
-            USER_EVENTS_ACTIONS.service_details.wasm_not_supported
-          );
-          sortBy = ReducerID.stdDev;
+        } else if (sortBy === 'outliers') {
+          return calculateOutlierValue(series, dataFrame);
         }
+      } catch (e) {
+        console.error(e);
+        // ML sorting panicked, fallback to stdDev
+        sortBy = ReducerID.stdDev;
       }
       const fieldReducer = fieldReducers.get(sortBy);
       const value =
@@ -61,6 +65,10 @@ export const sortSeries = memoize(
 );
 
 export const calculateDataFrameChangepoints = (data: DataFrame) => {
+  if (!wasmSupported()) {
+    throw new Error('WASM not supported, fall back to stdDev');
+  }
+
   const fields = data.fields.filter((f) => f.type === FieldType.number);
 
   const dataPoints = fields[0].values.length;
@@ -95,6 +103,54 @@ export const sortSeriesByName = (series: DataFrame[], direction: string) => {
   return sortedSeries;
 };
 
-const wasmSupported = () => {
-  return typeof WebAssembly === 'object';
+const initOutlierDetector = (series: DataFrame[]) => {
+  if (!wasmSupported()) {
+    return;
+  }
+
+  // Combine all frames into one by joining on time.
+  const joined = outerJoinDataFrames({ frames: series });
+  if (!joined) {
+    return;
+  }
+
+  // Get number fields: these are our series.
+  const joinedSeries = joined.fields.filter((f) => f.type === FieldType.number);
+  const nTimestamps = joinedSeries[0].values.length;
+  const points = new Float64Array(joinedSeries.flatMap((series) => series.values as number[]));
+
+  try {
+    const detector = OutlierDetector.dbscan({ sensitivity: 0.4 }).preprocess(points, nTimestamps);
+    outliers = detector.detect();
+  } catch (e) {
+    console.error(e);
+  }
+};
+
+let outliers: OutlierOutput | undefined = undefined;
+
+export const calculateOutlierValue = (series: DataFrame[], data: DataFrame): number => {
+  if (!wasmSupported()) {
+    throw new Error('WASM not supported, fall back to stdDev');
+  }
+  if (!outliers) {
+    throw new Error('Initialize outlier detector first');
+  }
+
+  const index = series.indexOf(data);
+  if (outliers.seriesResults[index].isOutlier) {
+    return outliers.seriesResults[index].outlierIntervals.length;
+  }
+
+  return 0;
+};
+
+export const wasmSupported = () => {
+  const support = typeof WebAssembly === 'object';
+
+  if (!support) {
+    reportAppInteraction(USER_EVENTS_PAGES.service_details, USER_EVENTS_ACTIONS.service_details.wasm_not_supported);
+  }
+
+  return support;
 };