From db79bbfd05226c81d28c9ca78620f117471132e3 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Fri, 2 Aug 2024 10:22:37 -0500 Subject: [PATCH] Service selection: Replacing datasource resource calls with query runners (#651) * chore: WIP - replacing index/volume with variable and sceneQueryRunner via runtime datasource --- cspell.config.json | 2 +- project-words.txt | 1 + provisioning/datasources/default.yaml | 4 +- provisioning/datasources/defaultCopy.yaml | 11 + .../Breakdowns/FieldsBreakdownScene.tsx | 6 +- .../Breakdowns/LabelBreakdownScene.tsx | 6 +- .../Breakdowns/Patterns/PatternNameLabel.tsx | 4 +- .../ServiceScene/LogsVolumePanel.tsx | 4 +- src/Components/ServiceScene/ServiceScene.tsx | 4 +- .../SelectServiceButton.tsx | 10 +- .../ServiceSelectionScene.tsx | 311 +++++++----------- src/services/datasource.ts | 116 +++++-- src/services/query.test.ts | 6 +- src/services/query.ts | 28 +- src/services/testIds.ts | 12 + src/services/variables.ts | 10 + tests/exploreServices.spec.ts | 78 ++++- tests/fixtures/explore.ts | 2 + tests/mocks/mockVolumeApiResponse.ts | 249 ++++++++++++++ 19 files changed, 618 insertions(+), 246 deletions(-) create mode 100644 provisioning/datasources/defaultCopy.yaml create mode 100644 tests/mocks/mockVolumeApiResponse.ts diff --git a/cspell.config.json b/cspell.config.json index b5e3c1824..6691d5ac3 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -1,5 +1,5 @@ { - "ignorePaths": ["node_modules", "/project-words.txt", "dist"], + "ignorePaths": ["node_modules", "/project-words.txt", "dist", "playwright-report"], "ignoreRegExpList": [], "words": [], "dictionaryDefinitions": [ diff --git a/project-words.txt b/project-words.txt index bb344a84b..73d4eea38 100644 --- a/project-words.txt +++ b/project-words.txt @@ -465,3 +465,4 @@ logspanel inital lezer logql +subqueries diff --git a/provisioning/datasources/default.yaml b/provisioning/datasources/default.yaml index 9414657b5..964020e9d 100644 --- a/provisioning/datasources/default.yaml +++ b/provisioning/datasources/default.yaml @@ -2,10 +2,10 @@ apiVersion: 1 datasources: - name: gdev-testdata - isDefault: false + isDefault: true type: testdata - name: gdev-loki type: loki uid: gdev-loki access: proxy - url: http://host.docker.internal:3100 \ No newline at end of file + url: http://host.docker.internal:3100 diff --git a/provisioning/datasources/defaultCopy.yaml b/provisioning/datasources/defaultCopy.yaml new file mode 100644 index 000000000..c2c14b7f1 --- /dev/null +++ b/provisioning/datasources/defaultCopy.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: gdev-testdata-copy + isDefault: false + type: testdata + - name: gdev-loki-copy + type: loki + uid: gdev-loki-copy + access: proxy + url: http://host.docker.internal:3100 diff --git a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx index 7b194b0b4..f25c7545c 100644 --- a/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.tsx @@ -25,7 +25,7 @@ import { Alert, Button, DrawStyle, LoadingPlaceholder, StackingMode, useStyles2 import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { getFilterBreakdownValueScene } from 'services/fields'; import { getQueryRunner, setLeverColorOverrides } from 'services/panel'; -import { buildLokiQuery } from 'services/query'; +import { buildDataQuery } from 'services/query'; import { ALL_VARIABLE_VALUE, getFieldGroupByVariable, @@ -283,7 +283,7 @@ export class FieldsBreakdownScene extends SceneObjectBase this.state.search.state.filter ?? ''; diff --git a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx index 36225e172..416c33807 100644 --- a/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx +++ b/src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.tsx @@ -22,7 +22,7 @@ import { Alert, Button, DrawStyle, LoadingPlaceholder, StackingMode, useStyles2 import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { DetectedLabel, getFilterBreakdownValueScene } from 'services/fields'; import { getQueryRunner, setLeverColorOverrides } from 'services/panel'; -import { buildLokiQuery } from 'services/query'; +import { buildDataQuery } from 'services/query'; import { ValueSlugs } from 'services/routing'; import { getLokiDatasource } from 'services/scenes'; import { ALL_VARIABLE_VALUE, getLabelGroupByVariable, VAR_LABEL_GROUP_BY, VAR_LABELS } from 'services/variables'; @@ -219,7 +219,7 @@ export class LabelBreakdownScene extends SceneObjectBase { .setUnit('short') .setData( getQueryRunner( - buildLokiQuery(getTimeSeriesExpr(this, LEVEL_VARIABLE_VALUE, false), { + buildDataQuery(getTimeSeriesExpr(this, LEVEL_VARIABLE_VALUE, false), { legendFormat: `{{${LEVEL_VARIABLE_VALUE}}}`, }) ) diff --git a/src/Components/ServiceScene/ServiceScene.tsx b/src/Components/ServiceScene/ServiceScene.tsx index eb01c7991..e0dbac567 100644 --- a/src/Components/ServiceScene/ServiceScene.tsx +++ b/src/Components/ServiceScene/ServiceScene.tsx @@ -17,7 +17,7 @@ import { Box, Stack, Tab, TabsBar, useStyles2 } from '@grafana/ui'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { DetectedLabel, DetectedLabelsResponse, updateParserFromDataFrame } from 'services/fields'; import { getQueryRunner } from 'services/panel'; -import { buildLokiQuery, renderLogQLStreamSelector } from 'services/query'; +import { buildDataQuery, renderLogQLStreamSelector } from 'services/query'; import { getDrilldownSlug, getDrilldownValueSlug, PageSlugs, PLUGIN_ID, ValueSlugs } from 'services/routing'; import { getExplorationFor, getLokiDatasource } from 'services/scenes'; import { @@ -90,7 +90,7 @@ export class ServiceScene extends SceneObjectBase { public constructor(state: MakeOptional) { super({ body: state.body ?? buildGraphScene(), - $data: getQueryRunner(buildLokiQuery(LOG_STREAM_SELECTOR_EXPR)), + $data: getQueryRunner(buildDataQuery(LOG_STREAM_SELECTOR_EXPR)), loading: true, ...state, }); diff --git a/src/Components/ServiceSelectionScene/SelectServiceButton.tsx b/src/Components/ServiceSelectionScene/SelectServiceButton.tsx index 27bd97c5f..05add1697 100644 --- a/src/Components/ServiceSelectionScene/SelectServiceButton.tsx +++ b/src/Components/ServiceSelectionScene/SelectServiceButton.tsx @@ -13,7 +13,7 @@ import { import { Button } from '@grafana/ui'; import { VariableHide } from '@grafana/schema'; import { addToFavoriteServicesInStorage } from 'services/store'; -import { getDataSourceVariable, getLabelsVariable } from 'services/variables'; +import { getDataSourceVariable, getLabelsVariable, getServiceSelectionStringVariable } from 'services/variables'; import { SERVICE_NAME, ServiceSelectionScene } from './ServiceSelectionScene'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { FilterOp } from 'services/filters'; @@ -63,11 +63,9 @@ export function selectService(service: string, sceneRef: SceneObject) { setParserIfFrameExistsForService(service, sceneRef); - const serviceSelectionScene = sceneGraph.getAncestor(sceneRef, ServiceSelectionScene); - // Setting the service variable state triggers a re-query of the services with invalid queries, so we clear out the body state to avoid triggering queries since - serviceSelectionScene.setState({ - servicesToQuery: undefined, - }); + const serviceSelectionVariable = getServiceSelectionStringVariable(sceneRef); + // Reset the service selection search to show all services + serviceSelectionVariable.changeValueTo(''); variable.setState({ filters: [ diff --git a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx index 09e0f6d61..8a5b84c74 100644 --- a/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx +++ b/src/Components/ServiceSelectionScene/ServiceSelectionScene.tsx @@ -1,17 +1,20 @@ import { css } from '@emotion/css'; -import { debounce, escapeRegExp } from 'lodash'; -import React, { useState } from 'react'; -import { DashboardCursorSync, GrafanaTheme2, TimeRange } from '@grafana/data'; +import { debounce } from 'lodash'; +import React from 'react'; +import { DashboardCursorSync, DataFrame, GrafanaTheme2, LoadingState, TimeRange, VariableHide } from '@grafana/data'; import { behaviors, PanelBuilders, SceneComponentProps, SceneCSSGridItem, SceneCSSGridLayout, + SceneDataProvider, sceneGraph, SceneObjectBase, SceneObjectState, + SceneQueryRunner, SceneVariable, + SceneVariableSet, VariableDependencyConfig, VizPanel, } from '@grafana/scenes'; @@ -25,37 +28,36 @@ import { StackingMode, useStyles2, } from '@grafana/ui'; -import { getLokiDatasource } from 'services/scenes'; import { getFavoriteServicesFromStorage } from 'services/store'; -import { getDataSourceVariable, getLabelsVariable, LEVEL_VARIABLE_VALUE, VAR_DATASOURCE } from 'services/variables'; +import { + getDataSourceVariable, + getLabelsVariable, + getServiceSelectionStringVariable, + LEVEL_VARIABLE_VALUE, + VAR_DATASOURCE, + VAR_SERVICE, + VAR_SERVICE_EXPR, +} from 'services/variables'; import { selectService, SelectServiceButton } from './SelectServiceButton'; -import { PLUGIN_ID } from 'services/routing'; -import { buildLokiQuery } from 'services/query'; +import { buildDataQuery, buildResourceQuery } from 'services/query'; import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics'; import { getQueryRunner, setLeverColorOverrides } from 'services/panel'; import { ConfigureVolumeError } from './ConfigureVolumeError'; import { NoVolumeError } from './NoVolumeError'; import { getLabelsFromSeries, toggleLevelFromFilter } from 'services/levels'; -import { isFetchError } from '@grafana/runtime'; import { ServiceFieldSelector } from '../ServiceScene/Breakdowns/FieldSelector'; +import { CustomConstantVariable } from '../../services/CustomConstantVariable'; +import { areArraysEqual } from '../../services/comparison'; export const SERVICE_NAME = 'service_name'; interface ServiceSelectionSceneState extends SceneObjectState { // The body of the component body: SceneCSSGridLayout; - // We query volume endpoint to get list of all services and order them by volume - servicesByVolume?: string[]; - // Keeps track of whether service list is being fetched from volume endpoint - isServicesByVolumeLoading: boolean; - // Keeps track of the search query in input field - searchServicesString: string; - // List of services to be shown in the body - servicesToQuery?: string[]; - // in case the volume api errors out - volumeApiError?: boolean; // Show logs of a certain level for a given service serviceLevel: Map; + // Logs volume API response as dataframe with SceneQueryRunner + $data: SceneDataProvider; } function getMetricExpression(service: string) { @@ -68,24 +70,25 @@ function getLogExpression(service: string, levelFilter: string) { export class ServiceSelectionScene extends SceneObjectBase { protected _variableDependency = new VariableDependencyConfig(this, { - // We want to subscribe to changes in datasource variables and update the top services when the datasource changes variableNames: [VAR_DATASOURCE], - onReferencedVariableValueChanged: async (variable: SceneVariable) => { - const { name } = variable.state; - if (name === VAR_DATASOURCE) { - // If datasource changes, we need to fetch services by volume for the new datasource - this.getServicesByVolume(); - } - }, + onReferencedVariableValueChanged: this.onReferencedVariableValueChanged.bind(this), }); constructor(state: Partial) { super({ body: new SceneCSSGridLayout({ children: [] }), - isServicesByVolumeLoading: false, - servicesByVolume: undefined, - searchServicesString: '', - servicesToQuery: undefined, + $variables: new SceneVariableSet({ + variables: [ + new CustomConstantVariable({ + name: VAR_SERVICE, + label: 'Service', + hide: VariableHide.hideVariable, + value: '', + skipUrlSync: true, + }), + ], + }), + $data: getQueryRunner(buildResourceQuery(`{${SERVICE_NAME}=~\`${VAR_SERVICE_EXPR}.+\`}`, 'volume')), serviceLevel: new Map(), ...state, }); @@ -93,6 +96,15 @@ export class ServiceSelectionScene extends SceneObjectBase { - // Updates servicesToQuery when servicesByVolume is changed - if (newState.servicesByVolume !== oldState.servicesByVolume) { - const ds = getDataSourceVariable(this).getValue()?.toString(); - let servicesToQuery: string[] = []; - if (ds && newState.servicesByVolume) { - servicesToQuery = createListOfServicesToQuery( - newState.servicesByVolume, - ds, - this.state.searchServicesString - ); - } - this.setState({ - servicesToQuery, - }); - } - - // Updates servicesToQuery when searchServicesString is changed - if (newState.searchServicesString !== oldState.searchServicesString) { - const ds = getDataSourceVariable(this).getValue()?.toString(); - let servicesToQuery: string[] = []; - if (ds && this.state.servicesByVolume) { - servicesToQuery = createListOfServicesToQuery( - this.state.servicesByVolume, - ds, - newState.searchServicesString - ); - } - this.setState({ - servicesToQuery, - }); - this.getServicesByVolume(newState.searchServicesString); - } - - // When servicesToQuery is changed, update the body and render the panels with the new services - if (newState.servicesToQuery !== oldState.servicesToQuery) { - this.updateBody(); - } - }) - ); + serviceVariable.changeValueTo(''); this._subs.add( - sceneGraph.getTimeRange(this).subscribeToState((newTime, oldTime) => { - if (shouldUpdateServicesByVolume(newTime.value, oldTime.value)) { - this.getServicesByVolume(); + this.state.$data.subscribeToState((newState, prevState) => { + // update body if the data is done loading, and the dataframes have changed + if ( + newState.data?.state === LoadingState.Done && + !areArraysEqual(prevState?.data?.series, newState?.data?.series) + ) { + this.updateBody(); } }) ); } - // Run to fetch services by volume - private async getServicesByVolume(service?: string) { - const timeRange = sceneGraph.getTimeRange(this).state.value; - this.setState({ - isServicesByVolumeLoading: true, - }); - const ds = await getLokiDatasource(this); - if (!ds) { - return; - } - - try { - const serviceSearch = service ? `(?i).*${escapeRegExp(service)}.*` : '.+'; - const volumeResponse = await ds.getResource( - 'index/volume', - { - query: `{${SERVICE_NAME}=~\`${serviceSearch}\`}`, - from: timeRange.from.utc().toISOString(), - to: timeRange.to.utc().toISOString(), - limit: 1000, - }, - { - headers: { - 'X-Query-Tags': `Source=${PLUGIN_ID}`, - }, - } - ); - const serviceMetrics: { [key: string]: number } = {}; - volumeResponse.data.result.forEach((item: any) => { - const serviceName = item['metric'][SERVICE_NAME]; - const value = Number(item['value'][1]); - serviceMetrics[serviceName] = value; - }); - - const servicesByVolume = Object.entries(serviceMetrics) - .sort((a, b) => b[1] - a[1]) // Sort by value in descending order - .map(([serviceName]) => serviceName); // Extract service names - - this.setState({ - volumeApiError: false, - servicesByVolume, - isServicesByVolumeLoading: false, - }); - } catch (error) { - console.log(`Failed to fetch top services:`, error); - const volumeApiError = isFetchError(error) && error.data.message?.includes('parse error') ? false : true; - this.setState({ - volumeApiError, - servicesByVolume: [], - isServicesByVolumeLoading: false, - }); - } - } - private updateBody() { + const { servicesToQuery } = this.getServices(this.state.$data.state.data?.series); // If no services are to be queried, clear the body - if (!this.state.servicesToQuery || this.state.servicesToQuery.length === 0) { + if (!servicesToQuery || servicesToQuery.length === 0) { this.state.body.setState({ children: [] }); } else { // If we have services to query, build the layout with the services. Children is an array of layouts for each service (1 row with 2 columns - timeseries and logs panel) - const children: SceneCSSGridItem[] = []; + const newChildren: SceneCSSGridItem[] = []; + const existingChildren: SceneCSSGridItem[] = this.state.body.state.children as SceneCSSGridItem[]; const timeRange = sceneGraph.getTimeRange(this).state.value; - for (const service of this.state.servicesToQuery) { - // for each service, we create a layout with timeseries and logs panel - children.push(this.buildServiceLayout(service, timeRange), this.buildServiceLogsLayout(service)); + + for (const service of servicesToQuery) { + const existing = existingChildren.filter((child) => { + const vizPanel = child.state.body as VizPanel | undefined; + return vizPanel?.state.title === service; + }); + + if (existing.length === 2) { + // If we already have grid items for this service, move them over to the new array of children, this will preserve their queryRunners, preventing duplicate queries from getting run + newChildren.push(existing[0], existing[1]); + } else { + // for each service, we create a layout with timeseries and logs panel + newChildren.push(this.buildServiceLayout(service, timeRange), this.buildServiceLogsLayout(service)); + } } + this.state.body.setState({ - children, + children: newChildren, isLazy: true, templateColumns: 'repeat(auto-fit, minmax(500px, 1fr) minmax(300px, 70vw))', autoRows: '200px', @@ -249,7 +179,8 @@ export class ServiceSelectionScene extends SceneObjectBase { const levelFilter = this.getLevelFilterForService(service); + // const timeRange = sceneGraph.getTimeRange(this).state.value; return new SceneCSSGridItem({ $behaviors: [new behaviors.CursorSync({ sync: DashboardCursorSync.Off })], body: PanelBuilders.logs() @@ -345,12 +276,14 @@ export class ServiceSelectionScene extends SceneObjectBase { - this.setState({ - searchServicesString: serviceString, - }); + public onSearchServicesChange = debounce((serviceString?: string) => { + const variable = getServiceSelectionStringVariable(this); + variable.changeValueTo(serviceString ?? ''); + reportAppInteraction( USER_EVENTS_PAGES.service_selection, USER_EVENTS_ACTIONS.service_selection.search_services_changed, @@ -373,29 +306,33 @@ export class ServiceSelectionScene extends SceneObjectBase) => { const styles = useStyles2(getStyles); - const { isServicesByVolumeLoading, servicesByVolume, servicesToQuery, body, volumeApiError } = model.useState(); + const { body, $data } = model.useState(); + const { data } = $data.useState(); - // searchQuery is used to keep track of the search query in input field - const [searchQuery, setSearchQuery] = useState(''); - const onSearchChange = (serviceName: string | undefined) => { - setSearchQuery(serviceName ?? ''); - model.onSearchServicesChange(serviceName ?? ''); + const serviceStringVariable = getServiceSelectionStringVariable(model); + const { value } = serviceStringVariable.useState(); + + const { servicesByVolume, servicesToQuery } = model.getServices(data?.series); + const isLogVolumeLoading = + data?.state === LoadingState.Loading || data?.state === LoadingState.Streaming || data === undefined; + const volumeApiError = $data.state.data?.state === LoadingState.Error; + + const onSearchChange = (serviceName: string) => { + model.onSearchServicesChange(serviceName); }; return (
{/** When services fetched, show how many services are we showing */} - {isServicesByVolumeLoading && ( - - )} - {!isServicesByVolumeLoading && <>Showing {servicesToQuery?.length ?? 0} services} + {isLogVolumeLoading && } + {!isLogVolumeLoading && <>Showing {servicesToQuery?.length ?? 0} services}
onSearchChange(serviceName ?? '')} selectOption={(value: string) => { selectService(value, model); }} @@ -409,9 +346,9 @@ export class ServiceSelectionScene extends SceneObjectBase {/** If we don't have any servicesByVolume, volume endpoint is probably not enabled */} - {!isServicesByVolumeLoading && volumeApiError && } - {!isServicesByVolumeLoading && !volumeApiError && !servicesByVolume?.length && } - {!isServicesByVolumeLoading && servicesToQuery && servicesToQuery.length > 0 && ( + {!isLogVolumeLoading && volumeApiError && } + {!isLogVolumeLoading && !volumeApiError && !servicesByVolume?.length && } + {servicesToQuery && servicesToQuery.length > 0 && (
@@ -420,6 +357,15 @@ export class ServiceSelectionScene extends SceneObjectBase ); }; + + private getServices(series?: DataFrame[]) { + const servicesByVolume: string[] = + series?.[0]?.fields?.find((field) => field.name === 'service_name')?.values ?? []; + const dsString = getDataSourceVariable(this).getValue()?.toString(); + const searchString = getServiceSelectionStringVariable(this).getValue(); + const servicesToQuery = createListOfServicesToQuery(servicesByVolume, dsString, String(searchString)); + return { servicesByVolume, servicesToQuery }; + } } // Create a list of services to query: @@ -440,41 +386,6 @@ function createListOfServicesToQuery(services: string[], ds: string, searchStrin return Array.from(new Set([...favoriteServicesToQuery, ...servicesToQuery])); } -function shouldUpdateServicesByVolume(newTime: TimeRange, oldTime: TimeRange) { - // Update if the time range is not within the same scope (hours vs. days) - if (newTime.to.diff(newTime.from, 'days') > 1 !== oldTime.to.diff(oldTime.from, 'days') > 1) { - return true; - } - // Update if the time range is less than 6 hours and the difference between the old and new 'from' and 'to' times is greater than 30 minutes - if (newTime.to.diff(newTime.from, 'hours') < 6 && timeDiffBetweenRangesLargerThan(newTime, oldTime, 'minutes', 30)) { - return true; - } - // Update if the time range is less than 1 day and the difference between the old and new 'from' and 'to' times is greater than 1 hour - if (newTime.to.diff(newTime.from, 'days') < 1 && timeDiffBetweenRangesLargerThan(newTime, oldTime, 'hours', 1)) { - return true; - } - // Update if the time range is more than 1 day and the difference between the old and new 'from' and 'to' times is greater than 1 day - if (newTime.to.diff(newTime.from, 'days') > 1 && timeDiffBetweenRangesLargerThan(newTime, oldTime, 'days', 1)) { - return true; - } - - return false; -} - -// Helper function to check if difference between two time ranges is larger than value -function timeDiffBetweenRangesLargerThan( - newTimeRange: TimeRange, - oldTimeRange: TimeRange, - unit: 'minutes' | 'hours' | 'days', - value: number -) { - const toChange = - newTimeRange.to.diff(oldTimeRange.to, unit) > value || newTimeRange.to.diff(oldTimeRange.to, unit) < -value; - const fromChange = - newTimeRange.from.diff(oldTimeRange.from, unit) > value || newTimeRange.from.diff(oldTimeRange.from, unit) < -value; - return toChange || fromChange; -} - function getStyles(theme: GrafanaTheme2) { return { container: css({ diff --git a/src/services/datasource.ts b/src/services/datasource.ts index 206fb31e3..b68053b53 100644 --- a/src/services/datasource.ts +++ b/src/services/datasource.ts @@ -1,16 +1,37 @@ -import { DataQueryRequest, DataQueryResponse, TestDataSourceResponse } from '@grafana/data'; -import { getDataSourceSrv } from '@grafana/runtime'; +import { createDataFrame, DataQueryRequest, DataQueryResponse, TestDataSourceResponse } from '@grafana/data'; +import { DataSourceWithBackend, getDataSourceSrv } from '@grafana/runtime'; import { RuntimeDataSource, SceneObject, sceneUtils } from '@grafana/scenes'; import { DataQuery } from '@grafana/schema'; -import { Observable, isObservable } from 'rxjs'; +import { Observable, Subscriber } from 'rxjs'; import { getDataSource } from './scenes'; +import { LokiQuery } from './query'; +import { PLUGIN_ID } from './routing'; export const WRAPPED_LOKI_DS_UID = 'wrapped-loki-ds-uid'; -type SceneDataQueryRequest = DataQueryRequest & { +export type SceneDataQueryRequest = DataQueryRequest & { scopedVars?: { __sceneObject?: { valueOf: () => SceneObject } }; }; +export type SceneDataQueryResourceRequest = { + resource: 'volume' | 'patterns' | 'detected_labels'; +}; +type TimeStampOfVolumeEval = number; +type VolumeCount = string; +type VolumeValue = [TimeStampOfVolumeEval, VolumeCount]; +type VolumeResult = { + metric: { + service_name: string; + }; + value: VolumeValue; +}; + +type IndexVolumeResponse = { + data: { + result: VolumeResult[]; + }; +}; + class WrappedLokiDatasource extends RuntimeDataSource { constructor(pluginId: string, uid: string) { super(pluginId, uid); @@ -25,26 +46,83 @@ class WrappedLokiDatasource extends RuntimeDataSource { getDataSourceSrv() .get(getDataSource(request.scopedVars.__sceneObject.valueOf())) .then((ds) => { - // override the target datasource to Loki - request.targets = request.targets.map((target) => { - target.datasource = ds; - return target; - }); - - // query the datasource and return either observable or promise - const dsResponse = ds.query(request); - if (isObservable(dsResponse)) { - dsResponse.subscribe(subscriber); - } else { - dsResponse.then((response) => { - subscriber.next(response); - subscriber.complete(); - }); + if (!(ds instanceof DataSourceWithBackend)) { + throw new Error('Invalid datasource!'); + } + + const requestType = request.targets?.[0]?.resource; + + switch (requestType) { + case 'volume': { + this.getVolume(request, ds, subscriber); + break; + } + case 'patterns': { + console.warn('not yet implemented'); + // this.transformPatternResponse(request, ds, subscriber); + break; + } + default: { + // override the target datasource to Loki + request.targets = request.targets.map((target) => { + target.datasource = ds; + return target; + }); + + // query the datasource and return either observable or promise + const dsResponse = ds.query(request); + dsResponse.subscribe(subscriber); + break; + } } }); }); } + private getVolume( + request: DataQueryRequest, + ds: DataSourceWithBackend, + subscriber: Subscriber + ) { + if (request.targets.length !== 1) { + throw new Error('Volume query can only have a single target!'); + } + + const targetsInterpolated = ds.interpolateVariablesInQueries(request.targets, request.scopedVars); + + const dsResponse = ds.getResource( + 'index/volume', + { + query: targetsInterpolated[0].expr, + from: request.range.from.utc().toISOString(), + to: request.range.to.utc().toISOString(), + limit: 1000, + }, + { + requestId: request.requestId ?? 'volume', + headers: { + 'X-Query-Tags': `Source=${PLUGIN_ID}`, + }, + } + ); + dsResponse.then((response: IndexVolumeResponse | undefined) => { + response?.data.result.sort((lhs: VolumeResult, rhs: VolumeResult) => { + const lVolumeCount: VolumeCount = lhs.value[1]; + const rVolumeCount: VolumeCount = rhs.value[1]; + return Number(rVolumeCount) - Number(lVolumeCount); + }); + // Scenes will only emit dataframes from the SceneQueryRunner, so for now we need to convert the API response to a dataframe + const df = createDataFrame({ + fields: [ + { name: 'service_name', values: response?.data.result.map((r) => r.metric.service_name) }, + { name: 'volume', values: response?.data.result.map((r) => Number(r.value[1])) }, + ], + }); + subscriber.next({ data: [df] }); + subscriber.complete(); + }); + } + testDatasource(): Promise { return Promise.resolve({ status: 'success', message: 'Data source is working', title: 'Success' }); } diff --git a/src/services/query.test.ts b/src/services/query.test.ts index 72a5a0f3f..6d8d08eea 100644 --- a/src/services/query.test.ts +++ b/src/services/query.test.ts @@ -1,7 +1,7 @@ -import { buildLokiQuery } from './query'; +import { buildDataQuery } from './query'; test('Given an expression outputs a Loki query', () => { - expect(buildLokiQuery('{place="luna"}')).toEqual({ + expect(buildDataQuery('{place="luna"}')).toEqual({ editorMode: 'code', expr: '{place="luna"}', queryType: 'range', @@ -11,7 +11,7 @@ test('Given an expression outputs a Loki query', () => { }); test('Given an expression and overrides outputs a Loki query', () => { - expect(buildLokiQuery('{place="luna"}', { editorMode: 'gpt', refId: 'C' })).toEqual({ + expect(buildDataQuery('{place="luna"}', { editorMode: 'gpt', refId: 'C' })).toEqual({ editorMode: 'gpt', expr: '{place="luna"}', queryType: 'range', diff --git a/src/services/query.ts b/src/services/query.ts index 9d2c93cfa..10b66edcb 100644 --- a/src/services/query.ts +++ b/src/services/query.ts @@ -1,6 +1,7 @@ import { AdHocVariableFilter, DataSourceApi } from '@grafana/data'; import { AppliedPattern } from 'Components/IndexScene/IndexScene'; import { PLUGIN_ID } from './routing'; +import { SceneDataQueryResourceRequest } from './datasource'; export type LokiQuery = { refId: string; @@ -13,7 +14,32 @@ export type LokiQuery = { datasource?: DataSourceApi; }; -export const buildLokiQuery = (expr: string, queryParamsOverrides?: Record): LokiQuery => { + +/** + * Builds the resource query + * @param expr string to be interpolated and executed in the resource request + * @param resource + * @param queryParamsOverrides + */ +export const buildResourceQuery = ( + expr: string, + resource: 'volume' | 'patterns' | 'detected_labels', + queryParamsOverrides?: Record +): LokiQuery & SceneDataQueryResourceRequest => { + return { + ...defaultQueryParams, + resource, + ...queryParamsOverrides, + expr, + }; +}; +/** + * Builds a loki data query + * @param expr + * @param queryParamsOverrides + * @returns LokiQuery + */ +export const buildDataQuery = (expr: string, queryParamsOverrides?: Record): LokiQuery => { return { ...defaultQueryParams, ...queryParamsOverrides, diff --git a/src/services/testIds.ts b/src/services/testIds.ts index 1752bf03c..ea4659d3d 100644 --- a/src/services/testIds.ts +++ b/src/services/testIds.ts @@ -8,6 +8,18 @@ export const testIds = { exploreServiceSearch: { search: 'data-testid search-services', }, + header: { + refreshPicker: 'data-testid RefreshPicker run button', + }, + variables: { + datasource: { + label: 'data-testid Dashboard template variables submenu Label Data source', + }, + serviceName: { + label: 'data-testid Dashboard template variables submenu Label service_name', + }, + }, + exploreServiceDetails: { searchLogs: 'data-testid search-logs', openExplore: 'data-testid open-explore', diff --git a/src/services/variables.ts b/src/services/variables.ts index 9e9a0942f..a29da31db 100644 --- a/src/services/variables.ts +++ b/src/services/variables.ts @@ -11,6 +11,8 @@ export const VAR_LEVELS = 'levels'; export const VAR_LEVELS_EXPR = '${levels}'; export const VAR_FIELD_GROUP_BY = 'fieldBy'; export const VAR_LABEL_GROUP_BY = 'labelBy'; +export const VAR_SERVICE = 'service'; +export const VAR_SERVICE_EXPR = '${service}'; export const VAR_DATASOURCE = 'ds'; export const VAR_DATASOURCE_EXPR = '${ds}'; export const VAR_LOGS_FORMAT = 'logsFormat'; @@ -92,3 +94,11 @@ export function getLogsFormatVariable(sceneRef: SceneObject) { } return variable; } + +export function getServiceSelectionStringVariable(sceneRef: SceneObject) { + const variable = sceneGraph.lookupVariable(VAR_SERVICE, sceneRef); + if (!(variable instanceof CustomConstantVariable)) { + throw new Error('VAR_SERVICE not found'); + } + return variable; +} diff --git a/tests/exploreServices.spec.ts b/tests/exploreServices.spec.ts index 452aa931e..525c1f228 100644 --- a/tests/exploreServices.spec.ts +++ b/tests/exploreServices.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from '@grafana/plugin-e2e'; import { ExplorePage } from './fixtures/explore'; -import {testIds} from "../src/services/testIds"; +import { testIds } from "../src/services/testIds"; +import { mockVolumeApiResponse } from "./mocks/mockVolumeApiResponse"; test.describe('explore services page', () => { let explorePage: ExplorePage; @@ -14,7 +15,16 @@ test.describe('explore services page', () => { test('should filter service labels on search', async ({ page }) => { await explorePage.servicesSearch.click(); await explorePage.servicesSearch.pressSequentially('mimir'); - await expect(page.getByTestId('data-testid Panel header mimir-ingester')).toBeVisible(); + // service name should be in time series panel + await expect(page.getByTestId('data-testid Panel header mimir-ingester').nth(0)).toBeVisible(); + // service name should also be in logs panel, just not visible to the user + await expect(page.getByTestId('data-testid Panel header mimir-ingester').nth(1)).toBeVisible(); + + // Exit out of the dropdown + await page.keyboard.press('Escape'); + // Only the first title is visible + await expect(page.getByText('mimir-ingester').nth(0)).toBeVisible() + await expect(page.getByText('mimir-ingester').nth(1)).not.toBeVisible() await expect(page.getByText('Showing 4 services')).toBeVisible(); }); @@ -22,4 +32,68 @@ test.describe('explore services page', () => { await explorePage.addServiceName(); await expect(explorePage.logVolumeGraph).toBeVisible(); }); + + test.describe('mock volume API calls', () => { + let logsVolumeCount: number, logsQueryCount: number; + + test.beforeEach(async ({page}) => { + logsVolumeCount = 0; + logsQueryCount = 0; + + await page.route('**/index/volume*', async route => { + const volumeResponse = mockVolumeApiResponse; + logsVolumeCount++ + + await route.fulfill({json: volumeResponse}) + }) + + await page.route('**/ds/query*', async route => { + logsQueryCount++ + await route.continue() + }) + + await Promise.all([ + page.waitForResponse(resp => resp.url().includes('index/volume')), + page.waitForResponse(resp => resp.url().includes('ds/query')), + ]); + }) + + test('refreshing time range should request panel data once', async ({page}) => { + expect(logsVolumeCount).toEqual(1) + expect(logsQueryCount).toEqual(4) + await explorePage.refreshPicker.click() + await explorePage.refreshPicker.click() + await explorePage.refreshPicker.click() + expect(logsVolumeCount).toEqual(4) + expect(logsQueryCount).toEqual(16) + }); + + test('navigating back will not re-run volume query', async ({page}) => { + expect(logsVolumeCount).toEqual(1) + expect(logsQueryCount).toEqual(4) + + await explorePage.addServiceName() + await page.getByTestId(testIds.variables.serviceName.label).click() + + expect(logsVolumeCount).toEqual(1) + // this should be 6, but there's an extra query being fired before the query expression can be interpolated + expect(logsQueryCount).toEqual(7) + + await explorePage.addServiceName() + await page.getByTestId(testIds.variables.serviceName.label).click() + + expect(logsVolumeCount).toEqual(1) + // Should be 8 + expect(logsQueryCount).toEqual(10) + + }) + + test('changing datasource will trigger new queries', async ({page}) => { + expect(logsVolumeCount).toEqual(1) + expect(logsQueryCount).toEqual(4) + await page.locator('div').filter({ hasText: /^gdev-loki$/ }).nth(1).click() + await page.getByText('gdev-loki-copy').click() + expect(logsVolumeCount).toEqual(2) + }) + }) }); diff --git a/tests/fixtures/explore.ts b/tests/fixtures/explore.ts index 2e61c3a63..97b7f1614 100644 --- a/tests/fixtures/explore.ts +++ b/tests/fixtures/explore.ts @@ -8,6 +8,7 @@ export class ExplorePage { servicesSearch: Locator; serviceBreakdownSearch: Locator; serviceBreakdownOpenExplore: Locator; + refreshPicker: Locator; constructor(public readonly page: Page) { this.firstServicePageSelect = this.page.getByText('Select').first(); @@ -15,6 +16,7 @@ export class ExplorePage { this.servicesSearch = this.page.getByTestId(testIds.exploreServiceSearch.search); this.serviceBreakdownSearch = this.page.getByTestId(testIds.exploreServiceDetails.searchLogs); this.serviceBreakdownOpenExplore = this.page.getByTestId(testIds.exploreServiceDetails.openExplore); + this.refreshPicker = this.page.getByTestId(testIds.header.refreshPicker) } async gotoServices() { diff --git a/tests/mocks/mockVolumeApiResponse.ts b/tests/mocks/mockVolumeApiResponse.ts new file mode 100644 index 000000000..2a8379aa2 --- /dev/null +++ b/tests/mocks/mockVolumeApiResponse.ts @@ -0,0 +1,249 @@ +export const mockVolumeApiResponse = { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "service_name": "tempo-distributor" + }, + "value": [ + 1722536046.066, + "53826521" + ] + }, + { + "metric": { + "service_name": "tempo-ingester" + }, + "value": [ + 1722536046.066, + "51585442" + ] + }, + { + "metric": { + "service_name": "mimir-ingester" + }, + "value": [ + 1722536046.066, + "2340497" + ] + }, + { + "metric": { + "service_name": "httpd" + }, + "value": [ + 1722536046.066, + "2093405" + ] + }, + { + "metric": { + "service_name": "nginx-json" + }, + "value": [ + 1722536046.066, + "1654774" + ] + }, + { + "metric": { + "service_name": "mimir-distributor" + }, + "value": [ + 1722536046.066, + "1516802" + ] + }, + { + "metric": { + "service_name": "mimir-querier" + }, + "value": [ + 1722536046.066, + "984900" + ] + }, + { + "metric": { + "service_name": "nginx" + }, + "value": [ + 1722536046.066, + "926310" + ] + }, + { + "metric": { + "service_name": "apache" + }, + "value": [ + 1722536046.066, + "874633" + ] + }, + { + "metric": { + "service_name": "mimir-ruler" + }, + "value": [ + 1722536046.066, + "301744" + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 0, + "linesProcessedPerSecond": 0, + "totalBytesProcessed": 0, + "totalLinesProcessed": 0, + "execTime": 0.375358251, + "queueTime": 0, + "subqueries": 0, + "totalEntriesReturned": 10, + "splits": 1, + "shards": 0, + "totalPostFilterLines": 0, + "totalStructuredMetadataBytesProcessed": 0 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 0, + "totalChunksMatched": 0, + "totalBatches": 0, + "totalLinesSent": 0, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0 + } + } + } +}