From b86fbc660e9a0b09f608e1a3d5541dde015e86a0 Mon Sep 17 00:00:00 2001 From: Roman Misyurin Date: Sun, 21 Apr 2024 23:42:06 +0200 Subject: [PATCH 1/2] Add traces and flamegraphs functionality --- .../flamegraph_and_tracing_support.json | 14 +- src/datasource/datasource.ts | 5 + src/datasource/sql_series.ts | 130 ++++++++++++++++++ .../QueryTextEditor/QueryTextEditor.tsx | 2 + 4 files changed, 145 insertions(+), 6 deletions(-) diff --git a/docker/grafana/dashboards/flamegraph_and_tracing_support.json b/docker/grafana/dashboards/flamegraph_and_tracing_support.json index 9d6dc7bee..55da4621f 100644 --- a/docker/grafana/dashboards/flamegraph_and_tracing_support.json +++ b/docker/grafana/dashboards/flamegraph_and_tracing_support.json @@ -20,6 +20,7 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "links": [], + "liveNow": false, "panels": [ { "datasource": { @@ -53,13 +54,13 @@ "dateTimeType": "TIMESTAMP", "editorMode": "builder", "extrapolate": false, - "format": "table", + "format": "traces", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, "query": "SELECT\r\n trace_id AS traceID,\r\n span_id AS spanID,\r\n operation_name AS operationName,\r\n parent_span_id AS parentSpanID,\r\n 'clickhouse' AS serviceName,\r\n intDiv(finish_time_us - start_time_us, 1000) AS duration,\r\n intDiv(start_time_us,1000) AS startTime,\r\n attribute AS tags,\r\n map('hostName',hostname) AS serviceTags\r\nFROM\r\n $table\r\nWHERE $timeFilter\r\nORDER BY traceID, startTime", "rawQuery": "/* grafana dashboard=FlameGraph and Tracing support, user=1 */ SELECT\r\n trace_id AS traceID,\r\n span_id AS spanID,\r\n operation_name AS operationName,\r\n parent_span_id AS parentSpanID,\r\n hostname AS serviceName,\r\n intDiv(finish_time_us - start_time_us, 1000) AS duration,\r\n intDiv(start_time_us,1000) AS startTime,\r\n attribute AS tags,\r\n map() AS serviceTags\r\nFROM\r\n system.opentelemetry_span_log\r\nWHERE finish_date >= toDate(1713432368) AND finish_date <= toDate(1713453968) AND intDiv(finish_time_us,1000000) >= 1713432368 AND intDiv(finish_time_us,1000000) <= 1713453968", "refId": "A", - "round": "", + "round": "0s", "showFormattedSQL": false, "showHelp": false, "skip_comments": false, @@ -112,14 +113,14 @@ "dateTimeType": "DATETIME", "editorMode": "builder", "extrapolate": false, - "format": "table", + "format": "flamegraph", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, "query": "SELECT length(trace) - level_num AS level, label, count() AS value, 0 self \r\nFROM $table \r\nARRAY JOIN arrayEnumerate(trace) AS level_num, \r\narrayMap(x -> if(addressToSymbol(x) != '', demangle(addressToSymbol(x)), 'unknown') , trace) AS label \r\nWHERE trace_type='Real' AND $timeFilter\r\nGROUP BY level, label, trace\r\nORDER BY trace, level\r\n", "rawQuery": "/* grafana dashboard=FlameGraph and Tracing support, user=1 */ SELECT level, label, count() AS value, 0 self \r\nFROM system.trace_log \r\nARRAY JOIN arrayEnumerate(trace) AS level, \r\narrayMap(x -> demangle(addressToSymbol(x) ), trace) AS label \r\nWHERE trace_type='Real' AND event_date >= toDate(1713685956) AND event_date <= toDate(1713707556) AND event_time >= toDateTime(1713685956) AND event_time <= toDateTime(1713707556)\r\nGROUP BY ALL;", "refId": "A", "round": "0s", - "showFormattedSQL": true, + "showFormattedSQL": false, "showHelp": false, "skip_comments": false, "step": "", @@ -130,19 +131,20 @@ "type": "flamegraph" } ], + "refresh": "", "schemaVersion": 39, "tags": [], "templating": { "list": [] }, "time": { - "from": "now-6h", + "from": "now-1m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "FlameGraph and Tracing support", "uid": "edimrzy0cijnkf", - "version": 1, + "version": 4, "weekStart": "" } diff --git a/src/datasource/datasource.ts b/src/datasource/datasource.ts index 232786fe7..c29a397a4 100644 --- a/src/datasource/datasource.ts +++ b/src/datasource/datasource.ts @@ -147,10 +147,15 @@ export class CHDataSource extends DataSourceApi { from: SqlQueryHelper.convertTimestamp(options.range.from), to: SqlQueryHelper.convertTimestamp(options.range.to), }); + if (target.format === 'table') { _.each(sqlSeries.toTable(), (data) => { result.push(data); }); + } else if (target.format === 'traces') { + result = sqlSeries.toTraces(); + } else if (target.format === 'flamegraph') { + result = sqlSeries.toFlamegraph(); } else if (target.format === 'logs') { result = sqlSeries.toLogs(); } else { diff --git a/src/datasource/sql_series.ts b/src/datasource/sql_series.ts index 5fb7fa5e0..b9795901b 100644 --- a/src/datasource/sql_series.ts +++ b/src/datasource/sql_series.ts @@ -1,6 +1,30 @@ import { each, find, isArray, omitBy, pickBy } from 'lodash'; import { DataFrame, FieldType, MutableDataFrame } from '@grafana/data'; +interface Trace { + traceID: string; + spanID: string; + parentSpanID?: string | null; + serviceName: string; + startTime: number; + duration: number; + operationName: string; + tags: object[]; + serviceTags: object[]; +} + +interface Field { + name: string; + type: string; + values: Array; + config: Record; +} + +interface TraceData { + fields: Field[]; + length: number; +} + export default class SqlSeries { refId: string; series: any; @@ -21,6 +45,112 @@ export default class SqlSeries { this.keys = options.keys || []; } + toTraces(): TraceData[] { + + let series: Trace[] = this.series; // Ensure 'this.series' is defined in the context where toTraces is called. + + function transformTraceData(inputData: Trace[]): TraceData[] { + const fields: { [key: string]: Field } = { + traceID: { name: 'traceID', type: 'string', values: [], config: {} }, + spanID: { name: 'spanID', type: 'string', values: [], config: {} }, + operationName: { name: 'operationName', type: 'string', values: [], config: {} }, + parentSpanID: { name: 'parentSpanID', type: 'string', values: [], config: {} }, + serviceName: { name: 'serviceName', type: 'string', values: [], config: {} }, + startTime: { name: 'startTime', type: 'number', values: [], config: {} }, + duration: { name: 'duration', type: 'number', values: [], config: {} }, + tags: { name: 'tags', type: 'number', values: [], config: {} }, + serviceTags: { name: 'serviceTags', type: 'number', values: [], config: {} }, + }; + + inputData.forEach(span => { + fields.traceID.values.push(span.traceID); + fields.spanID.values.push(span.spanID); + fields.operationName.values.push(span.operationName); + fields.parentSpanID.values.push(span.parentSpanID || null); // Assuming null if undefined + fields.serviceName.values.push(span.serviceName); + fields.startTime.values.push(parseInt(span.startTime.toString(), 10)); + fields.duration.values.push(parseInt(span.duration.toString(), 10)); + fields.tags.values.push(Object.entries(span.tags).map(([key, value]) => ({key: key, value: value}))); + fields.serviceTags.values.push(Object.entries(span.serviceTags).map(([key, value]) => ({key: key, value: value}))); + // Handle other fields if required + }); + + return [{ + fields: Object.values(fields), + length: inputData.length + }]; + } + + return transformTraceData(series); + } + + toFlamegraph(): any { + interface FlamegraphData { + label: string; + level: number; + value: string; + self: number; + } + + interface Field { + name: string; + type: string; + values: Array; + config: {}; + } + + try { + const series: FlamegraphData[] = this.series; + return transformTraceData(series); + } catch (error: any) { + return [{ + fields: [{ + name: 'error', + type: 'string', + values: [error?.message], + config: {} + }], + length: 1 + }]; + } + + function transformTraceData(inputData: FlamegraphData[]): any { + const sortedData = inputData.filter(item => { + return !(Number(item.level) === 0) + }) + + const fields: { [key: string]: Field } = { + label: { name: 'label', type: 'string', values: ['all'], config: {} }, + level: { name: 'level', type: 'number', values: [0], config: {} }, + value: { name: 'value', type: 'number', values: [0], config: {} }, + self: { name: 'self', type: 'number', values: [0], config: {} }, + }; + + const totalValue = inputData.filter(item => Number(item.level) === 1).reduce((acc, item) => { + return acc + Number(item.value); + }, 0); + + fields.value.values[0] = totalValue; + + sortedData.forEach(item => { + fields.label.values.push(item.label); + fields.level.values.push(Number(item.level)); + fields.value.values.push(Number(item.value)); + fields.self.values.push(item.self); + }); + + + console.log([{ + fields: Object.values(fields), + length: inputData.length + }]) + return [{ + fields: Object.values(fields), + length: inputData.length + }]; + } + } + toTable(): any { let self = this; let data: Array<{ columns: any[]; rows: any[]; type: string }> = []; diff --git a/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx b/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx index 663a7d6c0..e75a73c5d 100644 --- a/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx +++ b/src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx @@ -18,6 +18,8 @@ const FORMAT_OPTIONS = [ { label: 'Time series', value: 'time_series' }, { label: 'Table', value: 'table' }, { label: 'Logs', value: 'logs' }, + { label: 'Traces', value: 'traces' }, + { label: 'Flame Graph', value: 'flamegraph' }, ]; export const QueryTextEditor = ({ query, height, onEditorMount, onSqlChange, onFieldChange, formattedData, onRunQuery, datasource }: any) => { From 55ef3ac3e34b8ef10ae25e7cae3466dd68c71eab Mon Sep 17 00:00:00 2001 From: Roman Misyurin Date: Sun, 21 Apr 2024 23:47:34 +0200 Subject: [PATCH 2/2] remove logging --- src/datasource/sql_series.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/datasource/sql_series.ts b/src/datasource/sql_series.ts index b9795901b..dfa544328 100644 --- a/src/datasource/sql_series.ts +++ b/src/datasource/sql_series.ts @@ -139,11 +139,6 @@ export default class SqlSeries { fields.self.values.push(item.self); }); - - console.log([{ - fields: Object.values(fields), - length: inputData.length - }]) return [{ fields: Object.values(fields), length: inputData.length