From 8224556ffaa7cae0a8692a50006b9515d6e7f100 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 1 Jun 2022 15:38:37 +0400 Subject: [PATCH] Add allow sub-seconds time resolution with $timeSeriesMs and $timeFilterMs support, fix https://github.com/Altinity/clickhouse-grafana/issues/354, fix https://github.com/Altinity/clickhouse-grafana/issues/398 --- CHANGELOG.md | 1 + docker-compose.yaml | 1 + .../test_timeFilterMs_and_timeSeriesMs.json | 192 ++++++++++++++++++ pkg/eval_query.go | 55 ++++- pkg/eval_query_test.go | 42 +++- spec/sql_query_specs.jest.ts | 55 +++++ src/clickhouse-info.js | 24 ++- src/sql_query.ts | 49 ++++- 8 files changed, 396 insertions(+), 23 deletions(-) create mode 100644 docker/grafana/dashboards/test_timeFilterMs_and_timeSeriesMs.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b0912f7b..cec1e3d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Add support for Logs visualization, fix https://github.com/Altinity/clickhouse-grafana/issues/331, thanks @Fiery-Fenix and @pixelsquared * Add $conditionalTest to editor auto-complete * Add support $__searchFilter to template variable queries, fix https://github.com/Altinity/clickhouse-grafana/issues/354 +* Add allow sub-seconds time resolution with $timeSeriesMs and $timeFilterMs support, fix https://github.com/Altinity/clickhouse-grafana/issues/354, fix https://github.com/Altinity/clickhouse-grafana/issues/398 ## Fixes: * allow Nullable types in alert label name in backend part, fix https://github.com/Altinity/clickhouse-grafana/issues/405 diff --git a/docker-compose.yaml b/docker-compose.yaml index e5d57f7ca..cfbc9c8af 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -51,6 +51,7 @@ services: environment: GF_INSTALL_PLUGINS: grafana-piechart-panel,grafana-worldmap-panel GF_LOG_LEVEL: debug + # @todo wait grafana 9.0 implementation alerts provisioning https://github.com/grafana/grafana/issues/40983#issuecomment-1137770772 GF_UNIFIED_ALERTING_ENABLED: ${GF_UNIFIED_ALERTING_ENABLED:-false} GF_ALERTING_ENABLED: ${GF_ALERTING_ENABLED:-true} GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: vertamedia-clickhouse-datasource diff --git a/docker/grafana/dashboards/test_timeFilterMs_and_timeSeriesMs.json b/docker/grafana/dashboards/test_timeFilterMs_and_timeSeriesMs.json new file mode 100644 index 000000000..49b4f2bf1 --- /dev/null +++ b/docker/grafana/dashboards/test_timeFilterMs_and_timeSeriesMs.json @@ -0,0 +1,192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "max" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "name": "$timeSeriesMs and $timeFIlterMs alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "clickhouse", + "description": "fix https://github.com/Altinity/clickhouse-grafana/issues/344 and https://github.com/Altinity/clickhouse-grafana/issues/398", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.16", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "database": "default", + "dateColDataType": "", + "dateLoading": false, + "dateTimeColDataType": "d", + "dateTimeType": "DATETIME64", + "datetimeLoading": false, + "extrapolate": true, + "format": "time_series", + "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "interval": "", + "intervalFactor": 1, + "query": "SELECT\n $timeSeriesMs as t,\n count()\nFROM $table\n\nWHERE $timeFilterMs\n\nGROUP BY t\n\nORDER BY t\n", + "queryType": "randomWalk", + "rawQuery": "SELECT\n (intDiv(toFloat64(\"d\") * 1000, 20000) * 20000) as t,\n count()\nFROM default.test_datetime64\n\nWHERE \"d\" >= toDateTime64(1654061632511/1000, 3) AND \"d\" <= toDateTime64(1654083232511/1000, 3)\n\nGROUP BY t\n\nORDER BY t", + "refId": "A", + "round": "0s", + "skip_comments": true, + "table": "test_datetime64", + "tableLoading": false + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "visible": true + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$timeSeriesMs and $timeFIlterMs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:469", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:470", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "$timeFilterMs and $timeSeriesMs", + "uid": "iVjWIx97k", + "version": 2 +} \ No newline at end of file diff --git a/pkg/eval_query.go b/pkg/eval_query.go index 56d9d5d2b..a1f1c709e 100644 --- a/pkg/eval_query.go +++ b/pkg/eval_query.go @@ -12,8 +12,10 @@ import ( /* var NumberOnlyRegexp = regexp.MustCompile(`^[+-]?\d+(\.\d+)?$`) */ var timeSeriesMacroRegexp = regexp.MustCompile(`\$timeSeries\b`) +var timeSeriesMsMacroRegexp = regexp.MustCompile(`\$timeSeriesMs\b`) var naturalTimeSeriesMacroRegexp = regexp.MustCompile(`\$naturalTimeSeries\b`) var timeFilterMacroRegexp = regexp.MustCompile(`\$timeFilter\b`) +var timeFilterMsMacroRegexp = regexp.MustCompile(`\$timeFilterMs\b`) var tableMacroRegexp = regexp.MustCompile(`\$table\b`) var fromMacroRegexp = regexp.MustCompile(`\$from\b`) var toMacroRegexp = regexp.MustCompile(`\$to\b`) @@ -25,6 +27,7 @@ var timeFilter64ByColumnMacroRegexp = regexp.MustCompile(`\$timeFilter64ByColumn var fromMsMacroRegexp = regexp.MustCompile(`\$__from\b`) var toMsMacroRegexp = regexp.MustCompile(`\$__to\b`) +var intervalMsMacroRegexp = regexp.MustCompile(`\$__interval_ms\b`) type EvalQuery struct { RefId string `json:"refId"` @@ -40,6 +43,7 @@ type EvalQuery struct { IntervalFactor int `json:"intervalFactor"` Interval string `json:"interval"` IntervalSec int + IntervalMs int Database string `json:"database"` Table string `json:"table"` MaxDataPoints int64 @@ -66,12 +70,14 @@ func (q *EvalQuery) replace(query string) (string, error) { q.IntervalFactor = 1 } i := 1 * time.Second + ms := 1 * time.Millisecond if q.Interval != "" { duration, err := time.ParseDuration(q.Interval) if err != nil { return "", err } q.IntervalSec = int(math.Floor(duration.Seconds())) + q.IntervalMs = int(duration.Milliseconds()) } if q.IntervalSec <= 0 { if q.MaxDataPoints > 0 { @@ -79,10 +85,19 @@ func (q *EvalQuery) replace(query string) (string, error) { } else { i = q.To.Sub(q.From) / 100 } + if i > 1*time.Millisecond && q.IntervalMs <= 0 { + ms = i + } if i < 1*time.Second { i = 1 * time.Second } - q.IntervalSec, err = q.convertInterval(fmt.Sprintf("%fs", math.Floor(i.Seconds())), q.IntervalFactor) + q.IntervalSec, err = q.convertInterval(fmt.Sprintf("%fs", math.Floor(i.Seconds())), q.IntervalFactor, false) + if err != nil { + return "", err + } + } + if q.IntervalMs <= 0 { + q.IntervalMs, err = q.convertInterval(fmt.Sprintf("%dms", ms.Milliseconds()), q.IntervalFactor, true) if err != nil { return "", err } @@ -111,8 +126,10 @@ func (q *EvalQuery) replace(query string) (string, error) { } timeFilter := q.getDateTimeFilter(q.DateTimeType) + timeFilterMs := q.getDateTimeFilterMs(q.DateTimeType) if q.DateCol != "" { timeFilter = q.getDateFilter() + " AND " + timeFilter + timeFilterMs = q.getDateFilter() + " AND " + timeFilterMs } table := q.escapeIdentifier(q.Table) @@ -120,7 +137,7 @@ func (q *EvalQuery) replace(query string) (string, error) { table = q.escapeIdentifier(q.Database) + "." + table } - myRound, err := q.convertInterval(q.Round, q.IntervalFactor) + myRound, err := q.convertInterval(q.Round, q.IntervalFactor, false) if err != nil { return "", err } @@ -131,14 +148,19 @@ func (q *EvalQuery) replace(query string) (string, error) { to := q.convertTimestamp(q.round(q.To, myRound)) query = timeSeriesMacroRegexp.ReplaceAllString(query, strings.Replace(q.getTimeSeries(q.DateTimeType), "$", "$$", -1)) + query = timeSeriesMsMacroRegexp.ReplaceAllString(query, strings.Replace(q.getTimeSeriesMs(q.DateTimeType), "$", "$$", -1)) query = naturalTimeSeriesMacroRegexp.ReplaceAllString(query, strings.Replace(q.getNaturalTimeSeries(q.DateTimeType, from, to), "$", "$$", -1)) query = timeFilterMacroRegexp.ReplaceAllString(query, strings.Replace(timeFilter, "$", "$$", -1)) + query = timeFilterMsMacroRegexp.ReplaceAllString(query, strings.Replace(timeFilterMs, "$", "$$", -1)) query = tableMacroRegexp.ReplaceAllString(query, table) query = fromMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", from)) query = toMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", to)) + query = fromMsMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", q.From.UnixMilli())) + query = toMsMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", q.To.UnixMilli())) query = dateColMacroRegexp.ReplaceAllString(query, q.escapeIdentifier(q.DateCol)) query = dateTimeColMacroRegexp.ReplaceAllString(query, q.escapeIdentifier(q.DateTimeCol)) query = intervalMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", q.IntervalSec)) + query = intervalMsMacroRegexp.ReplaceAllString(query, fmt.Sprintf("%d", q.IntervalMs)) query = q.replaceTimeFilters(query, myRound) @@ -541,6 +563,16 @@ func (q *EvalQuery) getTimeSeries(dateTimeType string) string { return "(intDiv($dateTimeCol, $interval) * $interval) * 1000" } +func (q *EvalQuery) getTimeSeriesMs(dateTimeType string) string { + if dateTimeType == "DATETIME" { + return "(intDiv(toUInt32($dateTimeCol) * 1000, $__interval_ms) * $__interval_ms)" + } + if dateTimeType == "DATETIME64" { + return "(intDiv(toFloat64($dateTimeCol) * 1000, $__interval_ms) * $__interval_ms)" + } + return "(intDiv($dateTimeCol, $__interval_ms) * $__interval_ms)" +} + func (q *EvalQuery) getDateFilter() string { return "$dateCol >= toDate($from) AND $dateCol <= toDate($to)" } @@ -558,6 +590,19 @@ func (q *EvalQuery) getDateTimeFilter(dateTimeType string) string { return "$dateTimeCol >= " + convertFn("$from") + " AND $dateTimeCol <= " + convertFn("$to") } +func (q *EvalQuery) getDateTimeFilterMs(dateTimeType string) string { + convertFn := func(t string) string { + if dateTimeType == "DATETIME" { + return "toDateTime(" + t + ")" + } + if dateTimeType == "DATETIME64" { + return "toDateTime64(" + t + ", 3)" + } + return t + } + return "$dateTimeCol >= " + convertFn("$__from/1000") + " AND $dateTimeCol <= " + convertFn("$__to/1000") +} + func (q *EvalQuery) convertTimestamp(dt time.Time) int64 { return dt.UnixMilli() / 1000 } @@ -569,7 +614,7 @@ func (q *EvalQuery) round(dt time.Time, round int) time.Time { return dt.Truncate(time.Duration(round) * time.Second) } -func (q *EvalQuery) convertInterval(interval string, intervalFactor int) (int, error) { +func (q *EvalQuery) convertInterval(interval string, intervalFactor int, ms bool) (int, error) { if interval == "" { return 0, nil } @@ -577,8 +622,8 @@ func (q *EvalQuery) convertInterval(interval string, intervalFactor int) (int, e if err != nil { return 0, err } - if d < 1*time.Second { - d = 1 * time.Second + if ms { + return int(math.Ceil(float64(d.Milliseconds()) * float64(intervalFactor))), nil } return int(math.Ceil(d.Seconds() * float64(intervalFactor))), nil } diff --git a/pkg/eval_query_test.go b/pkg/eval_query_test.go index c200b1b19..c56f683bc 100644 --- a/pkg/eval_query_test.go +++ b/pkg/eval_query_test.go @@ -1202,7 +1202,7 @@ func TestEvalQueryTimeFilter64ByColumnAndRangeMs(t *testing.T) { } } -func TestEvalQueryTimeSeriesTimeFilsterAndDateTime64(t *testing.T) { +func TestEvalQueryTimeSeriesTimeFilterAndDateTime64(t *testing.T) { const description = "Query SELECT with $timeSeries $timeFilter and DATETIME64" const query = "SELECT $timeSeries as t, sum(x) AS metric\n" + "FROM $table\n" + @@ -1376,3 +1376,43 @@ func TestEvalQueryNaturalTimeSeries(t *testing.T) { r.Equal(expQuery, actualQuery, description) } + +/* check $timeSeriesMs $timeFilterMs https://github.com/Altinity/clickhouse-grafana/issues/344, https://github.com/Altinity/clickhouse-grafana/issues/398 */ +func TestEvalQueryTimeSeriesMsTimeFilterMsAndDateTime64(t *testing.T) { + const description = "Query SELECT with $timeSeriesMs $timeFilterMs and DATETIME64" + const query = "SELECT $timeSeriesMs as t, sum(x) AS metric\n" + + "FROM $table\n" + + "WHERE $timeFilterMs\n" + + "GROUP BY t\n" + + "ORDER BY t" + const expQuery = "SELECT (intDiv(toFloat64(\"d\") * 1000, 100) * 100) as t, sum(x) AS metric\n" + + "FROM default.test_datetime64\n" + + "WHERE \"d\" >= toDateTime64(1545613323200/1000, 3) AND \"d\" <= toDateTime64(1546300799200/1000, 3)\n" + + "GROUP BY t\n" + + "ORDER BY t" + + r := require.New(t) + from, err := time.Parse("2006-01-02 15:04:05.000Z", `2018-12-24 01:02:03.200Z`) + r.NoError(err) + to, err := time.Parse("2006-01-02 15:04:05.000Z", `2018-12-31 23:59:59.200Z`) + r.NoError(err) + + q := EvalQuery{ + Query: query, + From: from, + To: to, + Interval: "100ms", + IntervalFactor: 1, + SkipComments: false, + Table: "test_datetime64", + Database: "default", + DateTimeType: "DATETIME64", + DateCol: "", + DateTimeCol: "d", + Round: "100ms", + } + actualQuery, err := q.replace(query) + r.NoError(err) + + r.Equal(expQuery, actualQuery, description+" unexpected result") +} diff --git a/spec/sql_query_specs.jest.ts b/spec/sql_query_specs.jest.ts index d7e33514f..5b7ef24fb 100644 --- a/spec/sql_query_specs.jest.ts +++ b/spec/sql_query_specs.jest.ts @@ -1,6 +1,8 @@ import SqlQuery, {TimeRange} from '../src/sql_query'; import moment from "moment"; +// @ts-ignore import {RawTimeRangeStub} from './lib/raw_time_range_stub'; +// @ts-ignore import TemplateSrvStub from './lib/template_srv_stub'; describe("Query SELECT with $timeFilterByColumn and range with from and to:", () => { @@ -509,3 +511,56 @@ describe("check $naturalTimeSeries", () => { }); }); + +/* check $timeSeriesMs and $timeFilterMs https://github.com/Altinity/clickhouse-grafana/issues/344 */ +describe("Query SELECT with $timeSeriesMs $timeFilterMs and DATETIME64", () => { + const query = "SELECT $timeSeriesMs as t, sum(x) AS metric\n" + + "FROM $table\n" + + "WHERE $timeFilterMs\n" + + "GROUP BY t\n" + + "ORDER BY t"; + const expQuery = "SELECT (intDiv(toFloat64(\"d\") * 1000, 100) * 100) as t, sum(x) AS metric\n" + + "FROM default.test_datetime64\n" + + "WHERE \"d\" >= toDateTime64(1545613323200/1000, 3) AND \"d\" <= toDateTime64(1546300799200/1000, 3)\n" + + "GROUP BY t\n" + + "ORDER BY t"; + let templateSrv = new TemplateSrvStub(); + const adhocFilters = []; + let target = { + query: query, + interval: "100ms", + intervalFactor: 1, + skip_comments: false, + table: "test_datetime64", + database: "default", + dateTimeType: "DATETIME64", + dateColDataType: "", + dateTimeColDataType: "d", + round: "100ms", + rawQuery: "", + }; + const options = { + rangeRaw: { + from: moment('2018-12-24 01:02:03.200Z'), + to: moment('2018-12-31 23:59:59.200Z'), + }, + range: { + from: moment('2018-12-24 01:02:03.200Z'), + to: moment('2018-12-31 23:59:59.200Z'), + }, + scopedVars: { + __interval: { + text: "100ms", + value: "100ms", + }, + __interval_ms: { + text: "100", + value: 100, + }, + }, + }; + let sql_query = new SqlQuery(target, templateSrv, options); + it("applyMacros $timeSeriesMs with $timeFilterMs with DATETIME64", () => { + expect(sql_query.replace(options, adhocFilters)).toBe(expQuery); + }); +}); diff --git a/src/clickhouse-info.js b/src/clickhouse-info.js index f69f1e17a..ec90b9383 100644 --- a/src/clickhouse-info.js +++ b/src/clickhouse-info.js @@ -349,9 +349,11 @@ export default function () { "$to", "$interval", "$timeFilter", + "$timeFilterMs", "$timeFilterByColumn", "$timeFilter64ByColumn", "$timeSeries", + "$timeSeriesMs", "$naturalTimeSeries", "$rate", "$perSecond", @@ -2298,20 +2300,30 @@ export default function () { "def": "$timeFilter", "docText": "Replaced with currently selected `Time Range`. Requires `Column:Date` and `Column:DateTime` or `Column:TimeStamp` or `Column:DateTime64` to be selected" }, + { + "name": "$timeFilterMs", + "def": "$timeFilterMs", + "docText": "Replaced with currently selected `Time Range` with Millisecond precision. Requires `Column:DateTime64` to be selected" + }, { "name": "$timeFilterByColumn", "def": "$timeFilterByColumn(column_name)", "docText": "Replaced with currently selected `Time Range`. Requires column name with type `Date` and `DateTime` or `DateTime64`", }, - { - "name": "$timeFilter64ByColumn", - "def": "$timeFilter64ByColumn(column_name)", - "docText": "Replaced with currently selected `Time Range` with sub-seconds values. Requires column name with type `DateTime64`", - }, + { + "name": "$timeFilter64ByColumn", + "def": "$timeFilter64ByColumn(column_name)", + "docText": "Replaced with currently selected `Time Range` with sub-seconds values. Requires column name with type `DateTime64`", + }, { "name": "$timeSeries", "def": "$timeSeries", - "docText": "Replaced with special ClickHouse construction to convert results as time-series data. Use it as `SELECT $timeSeries...`. Require `Column:DateTime` or `Column:TimeStamp` to be selected" + "docText": "Replaced with special ClickHouse construction to convert results as time-series data. Use it as `SELECT $timeSeries...`. Require `Column:DateTime` or `Column:TimeStamp` or `Column:DateTime64` to be selected" + }, + { + "name": "$timeSeriesMs", + "def": "$timeSeriesMs", + "docText": "Replaced with special ClickHouse construction to convert results as time-series data with Milliseconds precision. Use it as `SELECT $timeSeriesMs...`. Require `Column:DateTime64` to be selected" }, { "name": "$naturalTimeSeries", diff --git a/src/sql_query.ts b/src/sql_query.ts index f3f48b796..86dfe3979 100644 --- a/src/sql_query.ts +++ b/src/sql_query.ts @@ -41,6 +41,7 @@ export default class SqlQuery { : 'DATETIME', i = this.templateSrv.replace(this.target.interval, options.scopedVars) || options.interval, interval = SqlQuery.convertInterval(i, this.target.intervalFactor || 1), + intervalMs = SqlQuery.convertInterval(i, this.target.intervalFactor || 1, true), adhocCondition = []; try { @@ -112,8 +113,10 @@ export default class SqlQuery { } query = SqlQuery.unescape(query); let timeFilter = SqlQuery.getDateTimeFilter(dateTimeType); + let timeFilterMs = SqlQuery.getDateTimeFilterMs(dateTimeType); if (typeof this.target.dateColDataType === "string" && this.target.dateColDataType.length > 0) { timeFilter = SqlQuery.getDateFilter() + ' AND ' + timeFilter; + timeFilterMs = SqlQuery.getDateFilter() + ' AND ' + timeFilterMs; } let table = SqlQuery.escapeIdentifier(this.target.table); @@ -127,14 +130,17 @@ export default class SqlQuery { this.target.rawQuery = query .replace(/\$timeSeries\b/g, SqlQuery.getTimeSeries(dateTimeType)) + .replace(/\$timeSeriesMs\b/g, SqlQuery.getTimeSeriesMs(dateTimeType)) .replace(/\$naturalTimeSeries/g, SqlQuery.getNaturalTimeSeries(dateTimeType, from, to)) .replace(/\$timeFilter\b/g, timeFilter) + .replace(/\$timeFilterMs\b/g, timeFilterMs) .replace(/\$table\b/g, table) .replace(/\$from\b/g, from) .replace(/\$to\b/g, to) .replace(/\$dateCol\b/g, SqlQuery.escapeIdentifier(this.target.dateColDataType)) .replace(/\$dateTimeCol\b/g, SqlQuery.escapeIdentifier(this.target.dateTimeColDataType)) .replace(/\$interval\b/g, interval) + .replace(/\$__interval_ms\b/g, intervalMs) .replace(/\$adhoc\b/g, renderedAdHocCondition); const round = this.target.round === "$step" @@ -517,6 +523,16 @@ export default class SqlQuery { return '(intDiv($dateTimeCol, $interval) * $interval) * 1000'; } + static getTimeSeriesMs(dateTimeType: string): string { + if (dateTimeType === 'DATETIME') { + return '(intDiv(toUInt32($dateTimeCol) * 1000, $__interval_ms) * $__interval_ms)'; + } + if (dateTimeType === 'DATETIME64') { + return '(intDiv(toFloat64($dateTimeCol) * 1000, $__interval_ms) * $__interval_ms)'; + } + return '(intDiv($dateTimeCol, $__interval_ms) * $__interval_ms)'; + } + static getDateFilter() { return '$dateCol >= toDate($from) AND $dateCol <= toDate($to)'; } @@ -534,6 +550,19 @@ export default class SqlQuery { return '$dateTimeCol >= ' + convertFn('$from') + ' AND $dateTimeCol <= ' + convertFn('$to'); } + static getDateTimeFilterMs(dateTimeType: string) { + let convertFn = function (t: string): string { + if (dateTimeType === 'DATETIME') { + return 'toDateTime(' + t + ')'; + } + if (dateTimeType === 'DATETIME64') { + return 'toDateTime64(' + t + ', 3)'; + } + return '(' + t + ')'; + }; + return '$dateTimeCol >= ' + convertFn('$__from/1000') + ' AND $dateTimeCol <= ' + convertFn('$__to/1000'); + } + // date is a moment object static convertTimestamp(date: any) { //return date.format("'Y-MM-DD HH:mm:ss'") @@ -558,7 +587,7 @@ export default class SqlQuery { return moment(rounded); } - static convertInterval(interval: any, intervalFactor: number): number { + static convertInterval(interval: any, intervalFactor: number, ms?: boolean): number { if (interval === undefined || typeof interval !== 'string' || interval === "") { return 0; } @@ -566,14 +595,12 @@ export default class SqlQuery { if (m === null) { throw {message: 'Received interval is invalid: ' + interval}; } - - let dur = moment.duration(parseInt(m[1]), m[2]); - let sec = dur.asSeconds(); - if (sec < 1) { - sec = 1; + let duration = moment.duration(parseInt(m[1]), m[2]); + let result = duration.asSeconds(); + if (ms) { + result = duration.asMilliseconds(); } - - return Math.ceil(sec * intervalFactor); + return Math.ceil(result * intervalFactor); } static interpolateQueryExpr(value, variable, defaultFormatFn) { @@ -699,7 +726,7 @@ export default class SqlQuery { static unescape(query) { - let macros = '$unescape('; + const macros = '$unescape('; let openMacros = query.indexOf(macros); while (openMacros !== -1) { let r = SqlQuery.betweenBraces(query.substring(openMacros + macros.length, query.length)); @@ -707,10 +734,10 @@ export default class SqlQuery { throw {message: '$unescape macros error: ' + r.error}; } let arg = r.result; - arg = arg.replace(/[']+/g, ''); + arg = arg.replace(/'+/g, ''); let closeMacros = openMacros + macros.length + r.result.length + 1; query = query.substring(0, openMacros) + arg + query.substring(closeMacros, query.length); - openMacros = query.indexOf('$unescape('); + openMacros = query.indexOf(macros); } return query; }