LogsVolumePanel: support filtering logs from the selected level (#474)

* feat(LogsVolumePanel): support filtering logs from the selected level

* chore(LogsVolumePanel): do nothing for AppendToSelection

* feat(Variables): introduce levels variable

* feat(IndexScene): add levels as a variable

* feat(AddToFiltersButton): use VAR_LEVELS for level filtering

* feat(LogsVolumePanel): use logs volume stream selector for queries

* test(AddToFiltersButton): update

* test(AddToFiltersButton): update

* feat(LogsVolumePanel): apply level visibility overrides

* fix(panel): fix override

* chore(routing): add new levels variable

* feat(Filters): add replaceFilter function

* fix(LogsVolumePanel): toggle or replace filters when clicking on level

* fix(panel): set __systemRef to hideSeriesFrom

* fix(ServiceScene): add VAR_LEVELS to variable dependency config

* chore: add space

* fix(LogsListScene): update handleIsFilterLabelActive to support VAR_LEVELS

* feat(LogsVolumePanel): toggle visibility in state subscription

* chore(LogsVolumePanel): store subscription

* chore: fix test

The whole thing is unmockable and it's not worth it

* chore: test e2e patch

* Revert "chore: test e2e patch"

This reverts commit 13df68d.

* chore: bad merge

* chore: try Grafana main image

* chore: fix e2e

* chore: fix e2e test

* fix(LabelBreakdownScene): add levels to queries

* chore: fix e2e

docker-compose.dev.yaml

@@ -10,7 +10,7 @@ services:
       context: ./.config
       args:
         grafana_image: ${GRAFANA_IMAGE:-grafana}
-        grafana_version: ${GRAFANA_VERSION:-11.0.0}
+        grafana_version: ${GRAFANA_VERSION:-latest}
     ports:
       - 3000:3000/tcp
     volumes:

src/Components/IndexScene/IndexScene.tsx

@@ -27,6 +27,7 @@ import {
   VAR_DATASOURCE,
   VAR_FIELDS,
   VAR_LABELS,
+  VAR_LEVELS,
   VAR_LINE_FILTER,
   VAR_LOGS_FORMAT,
   VAR_PATTERNS,
@@ -199,6 +200,21 @@ function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVari
     return operators;
   };
 
+  const levelsVariable = new AdHocFiltersVariable({
+    name: VAR_LEVELS,
+    label: 'Filters',
+    applyMode: 'manual',
+    layout: 'vertical',
+    getTagKeysProvider: () => Promise.resolve({ replace: true, values: [] }),
+    getTagValuesProvider: () => Promise.resolve({ replace: true, values: [] }),
+    expressionBuilder: renderLogQLFieldFilters,
+    hide: VariableHide.hideLabel,
+  });
+
+  levelsVariable._getOperators = () => {
+    return operators;
+  };
+
   const dsVariable = new DataSourceVariable({
     name: VAR_DATASOURCE,
     label: 'Data source',
@@ -216,6 +232,7 @@ function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVari
         dsVariable,
         labelVariable,
         fieldsVariable,
+        levelsVariable,
         // @todo where is patterns being added to the url? Why do we have var-patterns and patterns?
         new CustomVariable({
           name: VAR_PATTERNS,

src/Components/ServiceScene/Breakdowns/AddToFiltersButton.test.tsx

@@ -4,7 +4,7 @@ import { AddToFiltersButton, FilterType, addAdHocFilter, addToFilters } from './
 import { FieldType, createDataFrame } from '@grafana/data';
 import userEvent from '@testing-library/user-event';
 import { AdHocFiltersVariable, SceneObject, sceneGraph } from '@grafana/scenes';
-import { LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS } from 'services/variables';
+import { LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from 'services/variables';
 import { ServiceSceneState } from '../ServiceScene';
 
 describe('AddToFiltersButton', () => {
@@ -61,7 +61,7 @@ describe('AddToFiltersButton', () => {
     const lookup = jest.spyOn(sceneGraph, 'lookupVariable').mockReturnValue(new AdHocFiltersVariable({}));
     render(<button.Component model={button} />);
     userEvent.click(screen.getByRole('button', { name: 'Include' }));
-    await waitFor(async () => expect(lookup).toHaveBeenCalledWith(VAR_FIELDS, expect.anything()));
+    await waitFor(async () => expect(lookup).toHaveBeenCalledWith(VAR_LEVELS, expect.anything()));
   });
 });
 
@@ -211,7 +211,7 @@ describe('addToFilters and addAdHocFilter', () => {
       const lookupVariable = jest.spyOn(sceneGraph, 'lookupVariable').mockReturnValue(adHocVariable);
       addAdHocFilter({ key: LEVEL_VARIABLE_VALUE, value: 'info', operator: '=' }, {} as SceneObject, VAR_FIELDS);
 
-      expect(lookupVariable).toHaveBeenCalledWith(VAR_LABELS, expect.anything());
+      expect(lookupVariable).toHaveBeenCalledWith(VAR_LEVELS, expect.anything());
     });
   });
 });

src/Components/ServiceScene/Breakdowns/AddToFiltersButton.tsx

@@ -4,7 +4,7 @@ import { AdHocVariableFilter, DataFrame } from '@grafana/data';
 import { SceneObjectState, SceneObjectBase, SceneComponentProps, SceneObject, sceneGraph } from '@grafana/scenes';
 import { VariableHide } from '@grafana/schema';
 import { USER_EVENTS_ACTIONS, USER_EVENTS_PAGES, reportAppInteraction } from 'services/analytics';
-import { getAdHocFiltersVariable, LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS } from 'services/variables';
+import { getAdHocFiltersVariable, LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from 'services/variables';
 import { FilterButton } from 'Components/FilterButton';
 import { FilterOp } from 'services/filters';
 import { ServiceScene } from '../ServiceScene';
@@ -64,10 +64,36 @@ export function addToFilters(
   });
 }
 
+export function replaceFilter(
+  key: string,
+  value: string,
+  operator: Extract<FilterType, 'include' | 'exclude'>,
+  scene: SceneObject
+) {
+  const variable = getAdHocFiltersVariable(
+    validateVariableNameForField(key, resolveVariableNameForField(key, scene)),
+    scene
+  );
+  if (!variable) {
+    return;
+  }
+
+  variable.setState({
+    filters: [
+      {
+        key,
+        operator: operator === 'exclude' ? FilterOp.NotEqual : FilterOp.Equal,
+        value,
+      },
+    ],
+    hide: VariableHide.hideLabel,
+  });
+}
+
 function validateVariableNameForField(field: string, variableName: string) {
   // Special case: If the key is LEVEL_VARIABLE_VALUE, we need to use the VAR_FIELDS.
   if (field === LEVEL_VARIABLE_VALUE) {
-    return VAR_FIELDS;
+    return VAR_LEVELS;
   }
   return variableName;
 }

src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx

@@ -37,6 +37,8 @@ import {
   getLabelGroupByVariable,
   getLabelsVariable,
   getFieldsVariable,
+  VAR_LEVELS_EXPR,
+  getLevelsVariable,
 } from 'services/variables';
 import { ByFrameRepeater } from './ByFrameRepeater';
 import { FieldSelector } from './FieldSelector';
@@ -350,6 +352,7 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
   private getExpr(tagKey: string) {
     const labelsVariable = getLabelsVariable(this);
     const fieldsVariable = getFieldsVariable(this);
+    const levelsVariables = getLevelsVariable(this);
 
     let labelExpressionToAdd;
     let fieldExpressionToAdd = '';
@@ -365,9 +368,10 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
       .join(',');
 
     const fields = fieldsVariable.state.filters;
+    const levels = levelsVariables.state.filters;
     // if we have fields, we also need to add `VAR_LOGS_FORMAT_EXPR`
-    if (fields.length) {
-      return `sum(count_over_time({${streamSelectors}} ${fieldExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${tagKey})`;
+    if (fields.length || levels.length) {
+      return `sum(count_over_time({${streamSelectors}} ${fieldExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${VAR_LEVELS_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${tagKey})`;
     }
     return `sum(count_over_time({${streamSelectors}} ${fieldExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} [$__auto])) by (${tagKey})`;
   }

src/Components/ServiceScene/LogsListScene.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 
 import {
+  AdHocFiltersVariable,
   PanelBuilders,
   SceneComponentProps,
   SceneFlexItem,
@@ -21,7 +22,7 @@ import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '..
 import { DataFrame } from '@grafana/data';
 import { addToFilters, FilterType } from './Breakdowns/AddToFiltersButton';
 import { getLabelTypeFromFrame, LabelType } from 'services/fields';
-import { getAdHocFiltersVariable, VAR_FIELDS, VAR_LABELS } from 'services/variables';
+import { getAdHocFiltersVariable, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from 'services/variables';
 import { locationService } from '@grafana/runtime';
 import { LogOptionsScene } from './LogOptionsScene';
 import { getLogOption } from 'services/store';
@@ -159,18 +160,17 @@ export class LogsListScene extends SceneObjectBase<LogsListSceneState> {
   };
 
   public handleIsFilterLabelActive = (key: string, value: string) => {
-    const filters = getAdHocFiltersVariable(VAR_LABELS, this);
+    const labels = getAdHocFiltersVariable(VAR_LABELS, this);
     const fields = getAdHocFiltersVariable(VAR_FIELDS, this);
-    return (
-      (filters &&
-        filters.state.filters.findIndex(
-          (filter) => filter.operator === '=' && filter.key === key && filter.value === value
-        ) >= 0) ||
-      (fields &&
-        fields.state.filters.findIndex(
-          (filter) => filter.operator === '=' && filter.key === key && filter.value === value
-        ) >= 0)
-    );
+    const levels = getAdHocFiltersVariable(VAR_LEVELS, this);
+
+    const hasKeyValueFilter = (filter: AdHocFiltersVariable | null) =>
+      filter &&
+      filter.state.filters.findIndex(
+        (filter) => filter.operator === '=' && filter.key === key && filter.value === value
+      ) >= 0;
+
+    return hasKeyValueFilter(labels) || hasKeyValueFilter(fields) || hasKeyValueFilter(levels);
   };
 
   public handleFilterStringClick = (value: string) => {

src/Components/ServiceScene/LogsVolumePanel.tsx

@@ -1,10 +1,16 @@
 import React from 'react';
 
 import { PanelBuilders, SceneComponentProps, SceneObjectBase, SceneObjectState, VizPanel } from '@grafana/scenes';
-import { DrawStyle, LegendDisplayMode, StackingMode } from '@grafana/ui';
-import { getQueryRunner, setLeverColorOverrides } from 'services/panel';
+import { DrawStyle, LegendDisplayMode, PanelContext, SeriesVisibilityChangeMode, StackingMode } from '@grafana/ui';
+import { getQueryRunner, setLevelSeriesOverrides, setLeverColorOverrides } from 'services/panel';
 import { buildLokiQuery } from 'services/query';
-import { LEVEL_VARIABLE_VALUE, LOG_STREAM_SELECTOR_EXPR } from 'services/variables';
+import {
+  getAdHocFiltersVariable,
+  LEVEL_VARIABLE_VALUE,
+  LOG_VOLUME_STREAM_SELECTOR_EXPR,
+  VAR_LEVELS,
+} from 'services/variables';
+import { addToFilters, replaceFilter } from './Breakdowns/AddToFiltersButton';
 
 export interface LogsVolumePanelState extends SceneObjectState {
   panel?: VizPanel;
@@ -26,14 +32,14 @@ export class LogsVolumePanel extends SceneObjectBase<LogsVolumePanelState> {
   }
 
   private getVizPanel() {
-    return PanelBuilders.timeseries()
+    const viz = PanelBuilders.timeseries()
       .setTitle('Log volume')
       .setOption('legend', { showLegend: true, calcs: ['sum'], displayMode: LegendDisplayMode.List })
       .setUnit('short')
       .setData(
         getQueryRunner(
           buildLokiQuery(
-            `sum by (${LEVEL_VARIABLE_VALUE}) (count_over_time(${LOG_STREAM_SELECTOR_EXPR} | drop __error__ [$__auto]))`,
+            `sum by (${LEVEL_VARIABLE_VALUE}) (count_over_time(${LOG_VOLUME_STREAM_SELECTOR_EXPR} | drop __error__ [$__auto]))`,
             { legendFormat: `{{${LEVEL_VARIABLE_VALUE}}}` }
           )
         )
@@ -43,10 +49,63 @@ export class LogsVolumePanel extends SceneObjectBase<LogsVolumePanelState> {
       .setCustomFieldConfig('lineWidth', 0)
       .setCustomFieldConfig('pointSize', 0)
       .setCustomFieldConfig('drawStyle', DrawStyle.Bars)
-      .setOverrides(setLeverColorOverrides)
-      .build();
+      .setOverrides(setLeverColorOverrides);
+
+    const fieldFilters = getAdHocFiltersVariable(VAR_LEVELS, this);
+    const filteredLevels = fieldFilters?.state.filters.map((filter) => filter.value);
+    if (filteredLevels?.length) {
+      viz.setOverrides(setLevelSeriesOverrides.bind(null, filteredLevels));
+    }
+
+    const panel = viz.build();
+    panel.setState({
+      extendPanelContext: (_, context) => this.extendTimeSeriesLegendBus(context),
+    });
+
+    return panel;
   }
 
+  private extendTimeSeriesLegendBus = (context: PanelContext) => {
+    const originalOnToggleSeriesVisibility = context.onToggleSeriesVisibility;
+
+    const levelFilter = getAdHocFiltersVariable(VAR_LEVELS, this);
+    if (levelFilter) {
+      this._subs.add(
+        levelFilter?.subscribeToState((newState, prevState) => {
+          const hadLevel = prevState.filters.find((filter) => filter.key === LEVEL_VARIABLE_VALUE);
+          const removedLevel = newState.filters.findIndex((filter) => filter.key === LEVEL_VARIABLE_VALUE) < 0;
+          if (hadLevel && removedLevel) {
+            originalOnToggleSeriesVisibility?.(hadLevel.value, SeriesVisibilityChangeMode.ToggleSelection);
+          }
+          const addedLevel = newState.filters.find((filter) => filter.key === LEVEL_VARIABLE_VALUE);
+          if (addedLevel) {
+            originalOnToggleSeriesVisibility?.(addedLevel.value, SeriesVisibilityChangeMode.ToggleSelection);
+          }
+        })
+      );
+    }
+
+    context.onToggleSeriesVisibility = (level: string, mode: SeriesVisibilityChangeMode) => {
+      // @TODO. We don't yet support filters with multiple values.
+      if (mode === SeriesVisibilityChangeMode.AppendToSelection) {
+        return;
+      }
+
+      const levelFilter = getAdHocFiltersVariable(VAR_LEVELS, this);
+      if (!levelFilter) {
+        return;
+      }
+      const hadLevel = levelFilter.state.filters.find(
+        (filter) => filter.key === LEVEL_VARIABLE_VALUE && filter.value !== level
+      );
+      if (hadLevel) {
+        replaceFilter(LEVEL_VARIABLE_VALUE, level, 'include', this);
+      } else {
+        addToFilters(LEVEL_VARIABLE_VALUE, level, 'toggle', this);
+      }
+    };
+  };
+
   public static Component = ({ model }: SceneComponentProps<LogsVolumePanel>) => {
     const { panel } = model.useState();
     if (!panel) {

src/Components/ServiceScene/ServiceScene.tsx

@@ -29,6 +29,7 @@ import {
   VAR_DATASOURCE,
   VAR_FIELDS,
   VAR_LABELS,
+  VAR_LEVELS,
   VAR_PATTERNS,
 } from 'services/variables';
 import {
@@ -81,7 +82,7 @@ export interface ServiceSceneState extends SceneObjectState, ServiceSceneCustomS
 
 export class ServiceScene extends SceneObjectBase<ServiceSceneState> {
   protected _variableDependency = new VariableDependencyConfig(this, {
-    variableNames: [VAR_DATASOURCE, VAR_LABELS, VAR_FIELDS, VAR_PATTERNS],
+    variableNames: [VAR_DATASOURCE, VAR_LABELS, VAR_FIELDS, VAR_PATTERNS, VAR_LEVELS],
     onReferencedVariableValueChanged: this.onReferencedVariableValueChanged.bind(this),
   });
 
@@ -143,7 +144,7 @@ export class ServiceScene extends SceneObjectBase<ServiceSceneState> {
 
     if (this.state.$data) {
       this._subs.add(
-        this.state.$data?.subscribeToState((newState, prevState) => {
+        this.state.$data?.subscribeToState((newState) => {
           if (newState.data?.state === LoadingState.Done) {
             this.updateFields();
           }

src/services/panel.test.ts

@@ -10,6 +10,7 @@ describe('setLeverColorOverrides', () => {
     const overrides = {
       matchFieldsWithName: matchFieldsWithNameMock,
     };
+    // @ts-expect-error
     setLeverColorOverrides(overrides);
 
     expect(matchFieldsWithNameMock).toHaveBeenCalledTimes(5);

src/services/panel.ts

@@ -1,13 +1,12 @@
-import { DataFrame } from '@grafana/data';
-import { SceneDataTransformer, SceneQueryRunner } from '@grafana/scenes';
+import { DataFrame, FieldConfig, FieldMatcherID } from '@grafana/data';
+import { FieldConfigOverridesBuilder, SceneDataTransformer, SceneQueryRunner } from '@grafana/scenes';
 import { map, Observable } from 'rxjs';
 import { LokiQuery } from './query';
 import { EXPLORATION_DS } from './variables';
+import { HideSeriesConfig } from '@grafana/schema';
 
 const UNKNOWN_LEVEL_LOGS = 'logs';
-// TODO: `FieldConfigOverridesBuilder` is not exported, so it can not be used
-// here.
-export function setLeverColorOverrides(overrides: any) {
+export function setLeverColorOverrides(overrides: FieldConfigOverridesBuilder<FieldConfig>) {
   overrides.matchFieldsWithName('info').overrideColor({
     mode: 'fixed',
     fixedColor: 'semi-dark-green',
@@ -30,6 +29,32 @@ export function setLeverColorOverrides(overrides: any) {
   });
 }
 
+interface TimeSeriesFieldConfig extends FieldConfig {
+  hideFrom: HideSeriesConfig;
+}
+export function setLevelSeriesOverrides(levels: string[], overrideConfig: FieldConfigOverridesBuilder<FieldConfig>) {
+  overrideConfig
+    .match({
+      id: FieldMatcherID.byNames,
+      options: {
+        mode: 'exclude',
+        names: levels,
+        prefix: 'All except:',
+        readOnly: true,
+      },
+    })
+    .overrideCustomFieldConfig<TimeSeriesFieldConfig, 'hideFrom'>('hideFrom', {
+      legend: false,
+      tooltip: false,
+      viz: true,
+    });
+
+  // Setting __systemRef to hideSeriesFrom, allows the override to be changed by interacting with the viz
+  const overrides = overrideConfig.build();
+  // @ts-expect-error
+  overrides[overrides.length - 1].__systemRef = 'hideSeriesFrom';
+}
+
 export function sortLevelTransformation() {
   return (source: Observable<DataFrame[]>) => {
     return source.pipe(