Merge pull request #539 from Altinity/issue-508

Implement Flamegraphs and Traces functionality

docker/grafana/dashboards/flamegraph_and_tracing_support.json

@@ -20,6 +20,7 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "links": [],
+  "liveNow": false,
   "panels": [
     {
       "datasource": {
@@ -53,13 +54,13 @@
           "dateTimeType": "TIMESTAMP",
           "editorMode": "builder",
           "extrapolate": false,
-          "format": "table",
+          "format": "traces",
           "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
           "intervalFactor": 1,
           "query": "SELECT\r\n    trace_id AS traceID,\r\n    span_id AS spanID,\r\n    operation_name AS operationName,\r\n    parent_span_id AS parentSpanID,\r\n    'clickhouse' AS serviceName,\r\n    intDiv(finish_time_us - start_time_us, 1000) AS duration,\r\n    intDiv(start_time_us,1000) AS startTime,\r\n    attribute AS tags,\r\n    map('hostName',hostname) AS serviceTags\r\nFROM\r\n   $table\r\nWHERE $timeFilter\r\nORDER BY traceID, startTime",
           "rawQuery": "/* grafana dashboard=FlameGraph and Tracing support, user=1 */ SELECT\r\n    trace_id AS traceID,\r\n    span_id AS spanID,\r\n    operation_name AS operationName,\r\n    parent_span_id AS parentSpanID,\r\n    hostname AS serviceName,\r\n    intDiv(finish_time_us - start_time_us, 1000) AS duration,\r\n    intDiv(start_time_us,1000) AS startTime,\r\n    attribute AS tags,\r\n    map() AS serviceTags\r\nFROM\r\n   system.opentelemetry_span_log\r\nWHERE finish_date >= toDate(1713432368) AND finish_date <= toDate(1713453968) AND intDiv(finish_time_us,1000000) >= 1713432368 AND intDiv(finish_time_us,1000000) <= 1713453968",
           "refId": "A",
-          "round": "",
+          "round": "0s",
           "showFormattedSQL": false,
           "showHelp": false,
           "skip_comments": false,
@@ -112,14 +113,14 @@
           "dateTimeType": "DATETIME",
           "editorMode": "builder",
           "extrapolate": false,
-          "format": "table",
+          "format": "flamegraph",
           "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
           "intervalFactor": 1,
           "query": "SELECT length(trace)  - level_num AS level, label, count() AS value, 0 self \r\nFROM $table \r\nARRAY JOIN arrayEnumerate(trace) AS level_num, \r\narrayMap(x -> if(addressToSymbol(x) != '', demangle(addressToSymbol(x)), 'unknown') , trace) AS label \r\nWHERE trace_type='Real' AND $timeFilter\r\nGROUP BY level, label, trace\r\nORDER BY trace, level\r\n",
           "rawQuery": "/* grafana dashboard=FlameGraph and Tracing support, user=1 */ SELECT level, label, count() AS value, 0 self \r\nFROM system.trace_log \r\nARRAY JOIN arrayEnumerate(trace) AS level, \r\narrayMap(x -> demangle(addressToSymbol(x) ), trace) AS label \r\nWHERE trace_type='Real' AND event_date >= toDate(1713685956) AND event_date <= toDate(1713707556) AND event_time >= toDateTime(1713685956) AND event_time <= toDateTime(1713707556)\r\nGROUP BY ALL;",
           "refId": "A",
           "round": "0s",
-          "showFormattedSQL": true,
+          "showFormattedSQL": false,
           "showHelp": false,
           "skip_comments": false,
           "step": "",
@@ -130,19 +131,20 @@
       "type": "flamegraph"
     }
   ],
+  "refresh": "",
   "schemaVersion": 39,
   "tags": [],
   "templating": {
     "list": []
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-1m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "browser",
   "title": "FlameGraph and Tracing support",
   "uid": "edimrzy0cijnkf",
-  "version": 1,
+  "version": 4,
   "weekStart": ""
 }

src/datasource/datasource.ts

@@ -147,10 +147,15 @@ export class CHDataSource extends DataSourceApi<CHQuery, CHDataSourceOptions> {
           from: SqlQueryHelper.convertTimestamp(options.range.from),
           to: SqlQueryHelper.convertTimestamp(options.range.to),
         });
+
         if (target.format === 'table') {
           _.each(sqlSeries.toTable(), (data) => {
             result.push(data);
           });
+        } else if (target.format === 'traces') {
+          result = sqlSeries.toTraces();
+        } else if (target.format === 'flamegraph') {
+          result = sqlSeries.toFlamegraph();
         } else if (target.format === 'logs') {
           result = sqlSeries.toLogs();
         } else {

src/datasource/sql_series.ts

@@ -1,6 +1,30 @@
 import { each, find, isArray, omitBy, pickBy } from 'lodash';
 import { DataFrame, FieldType, MutableDataFrame } from '@grafana/data';
 
+interface Trace {
+  traceID: string;
+  spanID: string;
+  parentSpanID?: string | null;
+  serviceName: string;
+  startTime: number;
+  duration: number;
+  operationName: string;
+  tags: object[];
+  serviceTags: object[];
+}
+
+interface Field {
+  name: string;
+  type: string;
+  values: Array<string | number | null | object>;
+  config: Record<string, unknown>;
+}
+
+interface TraceData {
+  fields: Field[];
+  length: number;
+}
+
 export default class SqlSeries {
   refId: string;
   series: any;
@@ -21,6 +45,107 @@ export default class SqlSeries {
     this.keys = options.keys || [];
   }
 
+  toTraces(): TraceData[] {
+
+    let series: Trace[] = this.series; // Ensure 'this.series' is defined in the context where toTraces is called.
+
+    function transformTraceData(inputData: Trace[]): TraceData[] {
+      const fields: { [key: string]: Field } = {
+        traceID: { name: 'traceID', type: 'string', values: [], config: {} },
+        spanID: { name: 'spanID', type: 'string', values: [], config: {} },
+        operationName: { name: 'operationName', type: 'string', values: [], config: {} },
+        parentSpanID: { name: 'parentSpanID', type: 'string', values: [], config: {} },
+        serviceName: { name: 'serviceName', type: 'string', values: [], config: {} },
+        startTime: { name: 'startTime', type: 'number', values: [], config: {} },
+        duration: { name: 'duration', type: 'number', values: [], config: {} },
+        tags: { name: 'tags', type: 'number', values: [], config: {} },
+        serviceTags: { name: 'serviceTags', type: 'number', values: [], config: {} },
+      };
+
+      inputData.forEach(span => {
+        fields.traceID.values.push(span.traceID);
+        fields.spanID.values.push(span.spanID);
+        fields.operationName.values.push(span.operationName);
+        fields.parentSpanID.values.push(span.parentSpanID || null); // Assuming null if undefined
+        fields.serviceName.values.push(span.serviceName);
+        fields.startTime.values.push(parseInt(span.startTime.toString(), 10));
+        fields.duration.values.push(parseInt(span.duration.toString(), 10));
+        fields.tags.values.push(Object.entries(span.tags).map(([key, value]) => ({key: key, value: value})));
+        fields.serviceTags.values.push(Object.entries(span.serviceTags).map(([key, value]) => ({key: key, value: value})));
+        // Handle other fields if required
+      });
+
+      return [{
+        fields: Object.values(fields),
+        length: inputData.length
+      }];
+    }
+
+    return transformTraceData(series);
+  }
+
+  toFlamegraph(): any {
+    interface FlamegraphData {
+      label: string;
+      level: number;
+      value: string;
+      self: number;
+    }
+
+    interface Field {
+      name: string;
+      type: string;
+      values: Array<string | number>;
+      config: {};
+    }
+
+    try {
+      const series: FlamegraphData[] = this.series;
+      return transformTraceData(series);
+    } catch (error: any) {
+      return [{
+        fields: [{
+          name: 'error',
+          type: 'string',
+          values: [error?.message],
+          config: {}
+        }],
+        length: 1
+      }];
+    }
+
+    function transformTraceData(inputData: FlamegraphData[]): any {
+      const sortedData = inputData.filter(item => {
+        return !(Number(item.level) === 0)
+      })
+
+      const fields: { [key: string]: Field } = {
+        label: { name: 'label', type: 'string', values: ['all'], config: {} },
+        level: { name: 'level', type: 'number', values: [0], config: {} },
+        value: { name: 'value', type: 'number', values: [0], config: {} },
+        self: { name: 'self', type: 'number', values: [0], config: {} },
+      };
+
+      const totalValue = inputData.filter(item => Number(item.level) === 1).reduce((acc, item) => {
+        return acc + Number(item.value);
+      }, 0);
+
+      fields.value.values[0] = totalValue;
+
+      sortedData.forEach(item => {
+        fields.label.values.push(item.label);
+        fields.level.values.push(Number(item.level));
+        fields.value.values.push(Number(item.value));
+        fields.self.values.push(item.self);
+      });
+
+      return [{
+        fields: Object.values(fields),
+        length: inputData.length
+      }];
+    }
+  }
+
   toTable(): any {
     let self = this;
     let data: Array<{ columns: any[]; rows: any[]; type: string }> = [];

src/views/QueryEditor/components/QueryTextEditor/QueryTextEditor.tsx

@@ -18,6 +18,8 @@ const FORMAT_OPTIONS = [
   { label: 'Time series', value: 'time_series' },
   { label: 'Table', value: 'table' },
   { label: 'Logs', value: 'logs' },
+  { label: 'Traces', value: 'traces' },
+  { label: 'Flame Graph', value: 'flamegraph' },
 ];
 
 export const QueryTextEditor = ({ query, height, onEditorMount, onSqlChange, onFieldChange, formattedData, onRunQuery, datasource }: any) => {