diff --git a/.env b/.env index fe85b93..c4a616d 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ COMPOSE_DOCKER_CLI_BUILD=1 COMPOSE_FILE="docker/docker-compose.yml" COMPOSE_PROJECT_NAME="druidgrafana" -DRUID_VERSION="24.0.0" -GRAFANA_VERSION="9.1.6" +DRUID_VERSION="25.0.0" +GRAFANA_VERSION="9.3.6" POSTGRES_VERSION="14.5" ZOOKEEPER_VERSION="3.8.0" diff --git a/.gitignore b/.gitignore index c3a6d90..d437297 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ e2e-results/ #API key .api.key + +# Mac OS +.DS_Store diff --git a/README.md b/README.md index 9120235..87beae8 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,32 @@ Where `$VERSION` is for instance `1.0.0` and `$YOUR_PLUGIN_DIR` is for instance (Source: https://grafana.com/docs/grafana/latest/plugins/installation/) +### Build lolo-druid plugin on Mac and Install it via Kubernetes manually +#### Build lolo-druid plugin + +Tested on Mac Pro M1 - Monterey v12.6: + +* Launch Desktop Docker +* Build +```sh +./mage-macos buildAll +``` + +#### Copy the plugin into a Grafana Pod in Kubernetes + +* Make sure you have set a proper Kubernetes context +* Rename the `dist` folder into like `lolo-grafadruid-druid-datasource` +* Copy the plugin into a Grafana Pod in Kubernetes (e.g.,) +```sh +$ kubectl get pod -n +NAME READY STATUS RESTARTS AGE +grafana-7c549965bc-47trw 1/1 Running 0 1d + +$ kubectl cp -n ./dist grafana-7c549965bc-47trw:/var/lib/grafana/plugins/lolo-grafadruid-druid-datasource +``` +* Make sure your Grafana pod has the env variable: `GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS : lolo-grafadruid-druid-datasource` +* Restart the pod and check if lolo-druid is installed :) + ## Examples You can try out various advanced features of the plugin by importing the [demo dashboard](https://github.com/grafadruid/druid-grafana/blob/master/docker/grafana/dashboards/dashboard.json) and running it against the Wikipedia dataset used in the [Druid quickstart tutorial](https://druid.apache.org/docs/latest/tutorials/index.html#step-4-load-data). diff --git a/package.json b/package.json index 97e553b..eb7f868 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "druid-grafana", + "name": "loloco-druid-grafana", "version": "1.4.1", "description": "Connects Grafana to Druid", "scripts": { @@ -9,7 +9,7 @@ "dev": "grafana-toolkit plugin:dev", "watch": "grafana-toolkit plugin:dev --watch" }, - "author": "Grafadruid", + "author": "Loloco Grafadruid", "license": "Apache-2.0", "devDependencies": { "@emotion/css": "^11.10.0", @@ -22,7 +22,8 @@ "ace-builds": "^1.11.2", "react-ace": "^10.1.0", "react-datepicker": "^4.8.0", - "tslib": "^2.4.0" + "tslib": "^2.4.0", + "node-sql-parser": "^4.6.6" }, "engines": { "node": ">=16" diff --git a/src/ConfigEditor.tsx b/src/ConfigEditor.tsx index 31fbc96..0d47ec2 100644 --- a/src/ConfigEditor.tsx +++ b/src/ConfigEditor.tsx @@ -8,10 +8,20 @@ import { DruidConnectionSettings } from './configuration/ConnectionSettings'; import { ConnectionSettingsOptions } from './configuration/ConnectionSettings/types'; import { DruidQueryDefaultSettings } from './configuration/QuerySettings'; import { QuerySettingsOptions } from './configuration/QuerySettings/types'; +import { DruidAdhocSettings } from 'configuration/AdhocSettings'; +import { AdhocSettingsOptions } from 'configuration/AdhocSettings/types'; + +interface TabType { + label: string; + value: Tabs; + content: JSX.Element; + icon: IconName; +} enum Tabs { Connection, Query, + Adhoc, } interface Props extends DataSourcePluginOptionsEditorProps {} @@ -36,11 +46,7 @@ export class ConfigEditor extends PureComponent { const jsonData = { ...options.jsonData, ...connectionSettings }; const connectionSecretSettings = normalizeData(secretSettings, true, 'connection'); const secureJsonData = { ...options.secureJsonData, ...connectionSecretSettings }; - const connectionSecretSettingsFields = normalizeData( - secretSettingsFields, - true, - 'connection' - ) as KeyValue; + const connectionSecretSettingsFields = normalizeData(secretSettingsFields, true, 'connection') as KeyValue; const secureJsonFields = { ...options.secureJsonFields, ...connectionSecretSettingsFields }; onOptionsChange({ ...options, jsonData, secureJsonData, secureJsonFields }); }; @@ -53,6 +59,14 @@ export class ConfigEditor extends PureComponent { onOptionsChange({ ...options, jsonData }); }; + onAdhocOptionsChange = (adhocSettingsOptions: AdhocSettingsOptions) => { + const { onOptionsChange, options } = this.props; + const { settings } = adhocSettingsOptions; + const adhocSettings = normalizeData(settings, true, 'adhoc'); + const jsonData = { ...options.jsonData, ...adhocSettings }; + onOptionsChange({ ...options, jsonData }); + }; + connectionOptions = (): ConnectionSettingsOptions => { const { jsonData, secureJsonData, secureJsonFields } = this.props.options; return { @@ -69,24 +83,38 @@ export class ConfigEditor extends PureComponent { }; }; + adhocOptions = (): AdhocSettingsOptions => { + const { jsonData } = this.props.options; + return { + settings: normalizeData(jsonData, false, 'adhoc'), + }; + }; + render() { const connectionOptions = this.connectionOptions(); const queryOptions = this.queryOptions(); + const adhocOptions = this.adhocOptions(); - const ConnectionTab = { + const ConnectionTab: TabType = { label: 'Connection', value: Tabs.Connection, content: , icon: 'signal', }; - const QueryTab = { + const QueryTab: TabType = { label: 'Query defaults', value: Tabs.Query, content: , icon: 'database', }; + const AdhochTab: TabType = { + label: 'Adhoc variables', + value: Tabs.Adhoc, + content: , + icon: 'code-branch', + }; - const tabs = [ConnectionTab, QueryTab]; + const tabs = [ConnectionTab, QueryTab, AdhochTab]; const { activeTab } = this.state; return ( @@ -98,7 +126,7 @@ export class ConfigEditor extends PureComponent { label={t.label} active={t.value === activeTab} onChangeTab={() => this.onSelectTab(t)} - icon={t.icon as IconName} + icon={t.icon} /> ))} diff --git a/src/DruidDataSource.ts b/src/DruidDataSource.ts index 8eb786c..c5b55f9 100644 --- a/src/DruidDataSource.ts +++ b/src/DruidDataSource.ts @@ -1,34 +1,329 @@ import { DataSourceInstanceSettings, MetricFindValue, ScopedVars } from '@grafana/data'; -import { DataSourceWithBackend, getTemplateSrv } from '@grafana/runtime'; -import { DruidSettings, DruidQuery } from './types'; +import { DataSourceWithBackend, getTemplateSrv, logWarning } from '@grafana/runtime'; +import { SQL, type SqlExpression } from './util/SQL'; +import type { DruidSettings, DruidQuery, AdhocVariableModel, AdhocFilter } from './types'; const druidVariableRegex = /\"\[\[(\w+)(?::druid:(\w+))?\]\]\"|\"\${(\w+)(?::druid:(\w+))?}\"/g; export class DruidDataSource extends DataSourceWithBackend { settingsData: DruidSettings; + constructor(instanceSettings: DataSourceInstanceSettings) { super(instanceSettings); - this.settingsData = instanceSettings.jsonData; + + // This proxy is needed for the JS to actually adheer to the current TS type + this.settingsData = new Proxy(instanceSettings.jsonData, { + get(_, key: string) { + return Object.fromEntries( + Object.entries(instanceSettings.jsonData) + .filter((it) => it[0].startsWith(key)) + .map((it) => [it[0].substring(`${key}.`.length), it[1]]) + ); + }, + }); } + + /** + * Provides autocompletion for adhoc filter values given the chosen key + */ + async getTagValues(options: { key: string }): Promise { + if (this.settingsData.adhoc?.shouldNotAutocompleteValue) { + return []; + } + + const tableNames = ( + await this._postSqlQuery( + `SELECT "TABLE_NAME" FROM INFORMATION_SCHEMA.COLUMNS WHERE "TABLE_SCHEMA" = 'druid' AND "COLUMN_NAME" = '${options.key}'` + ) + ).map((it) => it.value); + + const completions = ( + await Promise.all( + tableNames.map(async (tableName) => + this._postSqlQuery( + this.settingsData.adhoc?.shouldNotLimitAutocompleteValue + ? `SELECT DISTINCT "${options.key}" FROM ${tableName}` + : `SELECT "${options.key}" FROM ${tableName} GROUP BY "${options.key}" ORDER BY COUNT("${options.key}") DESC LIMIT 1000` + ) + ) + ) + ).reduce((acc, it) => [...acc, ...it], []); + + return completions; + } + + /** + * Provides autocompletion for adhoc filter keys + */ + async getTagKeys() { + return this._postSqlQuery(`SELECT "COLUMN_NAME" FROM INFORMATION_SCHEMA.COLUMNS WHERE "TABLE_SCHEMA" = 'druid'`); + } + filterQuery(query: DruidQuery) { return !query.hide; } - applyTemplateVariables(templatedQuery: DruidQuery, scopedVars?: ScopedVars) { - const templateSrv = getTemplateSrv(); - let template = JSON.stringify({ ...templatedQuery, expr: undefined }).replace( - druidVariableRegex, - (match, variable1, format1, variable2, format2) => { - if (format1 || format2 === 'json') { - return '${' + (variable1 || variable2) + ':doublequote}'; + + _applyAdhocTemplateVariablesToSQL(templatedQuery: DruidQuery, adhocFilters: AdhocFilter[]) { + const query = SQL.parseSelectQuery(templatedQuery.builder.query); + + if (Array.isArray(query)) { + /* + * Maybe implement multi-query as syntactic sugar for UNION-ALL? + * UNION-ALL: + * + * Ex multi-query input: + * SELECT foo FROM bar, + * SELECT biz FROM boz + * + * Ex UNION-ALL result: + * SELECT foo FROM bar + * UNION ALL + * SELECT biz FROM boz + */ + throw new Error('Multi-query not yet supported'); + } + + const initialWhere = (() => { + if (query.where) { + return query.where; + } + const firstAdhoc = adhocFilters.pop(); + return { + type: 'binary_expr', + operator: firstAdhoc?.operator, + left: { + type: 'column_ref', + table: null, + column: firstAdhoc?.key, + }, + right: { + type: 'single_quote_string', + value: firstAdhoc?.value, + }, + } as SqlExpression; + })(); + + query.where = adhocFilters.reduce((acc, filter) => { + const expression = ((): SqlExpression => { + if (filter.operator === '<' || filter.operator === '>') { + return { + type: 'binary_expr', + operator: filter.operator, + left: { + type: 'column_ref', + table: null, + column: filter.key, + }, + right: + typeof filter.value === 'number' + ? { + type: 'number', + value: filter.value, + } + : { + type: 'single_quote_string', + value: String(filter.value), + }, + }; + } else if (filter.operator === '!~') { + return { + type: 'binary_expr', + operator: 'NOT LIKE', + left: { + type: 'column_ref', + table: null, + column: filter.key, + }, + right: { + type: 'single_quote_string', + value: `%${filter.value}%`, + }, + }; + } else if (filter.operator === '=~') { + return { + type: 'binary_expr', + operator: 'LIKE', + left: { + type: 'column_ref', + table: null, + column: filter.key, + }, + right: { + type: 'single_quote_string', + value: `%${filter.value}%`, + }, + }; } - return match; + return { + type: 'binary_expr', + operator: filter.operator, + left: { + type: 'column_ref', + table: null, + column: filter.key, + }, + right: { + type: 'single_quote_string', + value: String(filter.value), + }, + }; + })(); + + return { + type: 'binary_expr', + operator: 'AND', + left: acc, + right: expression, + }; + }, initialWhere); + + return { + ...templatedQuery, + builder: { + ...templatedQuery.builder, + query: SQL.stringify(query), + }, + }; + } + + _applyAdhocTemplateVariablesToNativeQuery(templatedQuery: DruidQuery, adhocFilters: AdhocFilter[]) { + return { + ...templatedQuery, + builder: { + ...templatedQuery.builder, + filter: { + type: 'and', + fields: [ + templatedQuery.builder.filter, + ...adhocFilters.map((it) => { + switch (it.operator) { + case '=': + return { + type: 'selector', + dimension: it.key, + value: it.value, + }; + case '!=': + return { + type: 'not', + field: { + type: 'selector', + dimension: it.key, + value: it.value, + }, + }; + case '<': + return { + type: 'bound', + ordering: 'numeric', + dimension: it.key, + upper: it.value, + }; + case '>': + return { + type: 'bound', + ordering: 'numeric', + dimension: it.key, + lower: it.value, + }; + case '=~': + return { + type: 'like', + dimension: it.key, + pattern: `%${it.value}%`, + }; + case '!~': + return { + type: 'not', + field: { + type: 'like', + dimension: it.key, + value: `%${it.value}%`, + }, + }; + } + }), + ], + }, + }, + }; + } + + _applyAdhocTemplateVariables(templatedQuery: DruidQuery) { + const adhocFilters = getTemplateSrv() + .getVariables() + .filter((it): it is AdhocVariableModel => it.type === 'adhoc') + .filter((it) => it.datasource && it.datasource.uid === this.uid) + .reduce((acc, it) => [...acc, ...it.filters], []) + .filter((it) => it.value !== undefined) + .filter((it) => { + if (['<', '>', '=', '!=', '=~', '!~'].includes(it.operator)) { + return true; + } + logWarning(`Skipping unexpected filter operator: ${it.operator}`); + return false; + }); + + if (adhocFilters.length === 0) { + return templatedQuery; + } + + switch (templatedQuery.builder.queryType) { + case 'sql': + return this._applyAdhocTemplateVariablesToSQL(templatedQuery, adhocFilters); + case 'timeseries': + case 'topN': + case 'groupBy': + case 'scan': + case 'search': + case 'timeBoundary': + return this._applyAdhocTemplateVariablesToNativeQuery(templatedQuery, adhocFilters); + default: + return templatedQuery; + } + } + + applyTemplateVariables(templatedQuery: DruidQuery, scopedVars?: ScopedVars) { + const template = JSON.stringify({ + ...templatedQuery, + expr: undefined, + }).replace(druidVariableRegex, (match, variable1, format1, variable2, format2) => { + if (format1 || format2 === 'json') { + return '${' + (variable1 || variable2) + ':doublequote}'; } - ); - return { ...JSON.parse(templateSrv.replace(template, scopedVars)), expr: templatedQuery.expr }; + return match; + }); + return this._applyAdhocTemplateVariables({ + ...JSON.parse(getTemplateSrv().replace(template, scopedVars)), + expr: templatedQuery.expr, + }); } + async metricFindQuery(query: DruidQuery, options?: any): Promise { - return this.postResource('query-variable', this.applyTemplateVariables(query)).then((response) => { - return response; + return this.postResource('query-variable', this.applyTemplateVariables(query)); + } + + async _postSqlQuery(queryString: string): Promise> { + const query: DruidQuery = { + builder: { + queryType: 'sql', + query: queryString, + }, + settings: { + contextParameters: [ + { + name: 'AA', + value: 'BB', + }, + ], + format: 'long', + }, + expr: '', + refId: '', + }; + return this.postResource('query-variable', { + ...query, + expr: JSON.stringify(query), }); } } diff --git a/src/configuration/AdhocSettings/DruidAdhocSettings.tsx b/src/configuration/AdhocSettings/DruidAdhocSettings.tsx new file mode 100644 index 0000000..c89cea7 --- /dev/null +++ b/src/configuration/AdhocSettings/DruidAdhocSettings.tsx @@ -0,0 +1,55 @@ +import React, { ChangeEvent } from 'react'; +import { FieldSet, Field, Switch } from '@grafana/ui'; +import { css } from '@emotion/css'; +import { AdhocSettings, AdhocSettingsProps } from './types'; + +export const DruidAdhocSettings = (props: AdhocSettingsProps) => { + const { options, onOptionsChange } = props; + const { settings } = options; + + const onSettingChange = (event: ChangeEvent) => { + switch (event.target.name as keyof AdhocSettings) { + case 'shouldNotAutocompleteValue': + settings.shouldNotAutocompleteValue = !event.currentTarget.checked; + break; + case 'shouldNotLimitAutocompleteValue': + settings.shouldNotLimitAutocompleteValue = !event.currentTarget.checked; + break; + } + onOptionsChange({ ...options, settings: settings }); + }; + + return ( + <> +
+ + + + {!settings.shouldNotAutocompleteValue && ( + <> + + + + + )} +
+ + ); +}; diff --git a/src/configuration/AdhocSettings/index.ts b/src/configuration/AdhocSettings/index.ts new file mode 100644 index 0000000..edda8e7 --- /dev/null +++ b/src/configuration/AdhocSettings/index.ts @@ -0,0 +1 @@ +export { DruidAdhocSettings } from './DruidAdhocSettings'; diff --git a/src/configuration/AdhocSettings/types.ts b/src/configuration/AdhocSettings/types.ts new file mode 100644 index 0000000..88cb45a --- /dev/null +++ b/src/configuration/AdhocSettings/types.ts @@ -0,0 +1,13 @@ +export interface AdhocSettings { + shouldNotAutocompleteValue?: boolean; + shouldNotLimitAutocompleteValue?: boolean; +} + +export interface AdhocSettingsOptions { + settings: AdhocSettings; +} + +export interface AdhocSettingsProps { + options: AdhocSettingsOptions; + onOptionsChange: (options: AdhocSettingsOptions) => void; +} diff --git a/src/plugin.json b/src/plugin.json index cdfb2a9..8fbf838 100644 --- a/src/plugin.json +++ b/src/plugin.json @@ -1,16 +1,16 @@ { "type": "datasource", - "name": "Druid", - "id": "grafadruid-druid-datasource", + "name": "Lolo Druid", + "id": "loloco-grafadruid-druid-datasource", "metrics": true, "backend": true, "alerting": true, "logs": true, - "executable": "grafadruid-druid-datasource", + "executable": "loloco-grafadruid-druid-datasource", "info": { "description": "Connects Grafana to Druid", "author": { - "name": "Grafadruid" + "name": "Loloco Grafadruid" }, "keywords": ["druid"], "logos": { diff --git a/src/types.ts b/src/types.ts index b5e8da9..e237033 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,7 @@ -import { DataQuery, DataSourceJsonData } from '@grafana/data'; +import { DataQuery, DataSourceJsonData, MetricFindValue, VariableModel } from '@grafana/data'; import { QuerySettings } from './configuration/QuerySettings/types'; import { ConnectionSettings } from './configuration/ConnectionSettings/types'; +import { AdhocSettings } from 'configuration/AdhocSettings/types'; //expr is a workaround: https://github.com/grafana/grafana/issues/30013 export interface DruidQuery extends DataQuery { @@ -12,6 +13,32 @@ export interface DruidQuery extends DataQuery { export interface DruidSettings extends DataSourceJsonData { connection?: ConnectionSettings; query?: QuerySettings; + adhoc?: AdhocSettings; } export interface DruidSecureSettings {} + +export interface AdhocFilter { + key: string; + operator: '=' | '!=' | '<' | '>' | '=~' | '!~'; + value: MetricFindValue['value']; + condition: unknown; +} + +export interface AdhocVariableModel extends VariableModel { + type: 'adhoc'; + id: string; + datasource: { + type: string; + uid: string; + }; + filters: AdhocFilter[]; + hide: number; + skipUrlSync: boolean; + rootStateKey: string; + global: boolean; + index: number; + state: string; + error: unknown; + description: unknown; +} diff --git a/src/util/SQL.ts b/src/util/SQL.ts new file mode 100644 index 0000000..7a0943c --- /dev/null +++ b/src/util/SQL.ts @@ -0,0 +1,76 @@ +import * as NodeSqlParser from 'node-sql-parser'; + +export const SQL = (() => { + const parser = new NodeSqlParser.Parser(); + const options = { + database: 'postgresql', + }; + + const api = { + parse: (queryString: string) => parser.astify(queryString, options), + parseSelectQuery: (queryString: string) => { + const query = api.parse(queryString); + if (Array.isArray(query)) { + if (query.some((it) => it.type !== 'select')) { + throw Error('Unexpected non-select query type'); + } + return query as SqlSelectQuery[]; + } + if (query.type !== 'select') { + throw Error('Unexpected non-select query type'); + } + return query as SqlSelectQuery; + }, + stringify: (query: NodeSqlParser.AST) => parser.sqlify(query, options), + }; + return api; +})(); + +interface SqlColumnRef { + type: 'column_ref'; + table: string | null; + column: string; +} + +interface SqlStringValue { + type: 'single_quote_string'; + value: string; +} + +interface SqlNullValue { + type: 'null'; + value: null; +} + +interface SqlNumberValue { + type: 'number'; + value: number; +} + +interface SqlUnaryExpression { + type: 'unary_expr'; + operator: string; + expr: SqlExpression; +} + +type SqlBinaryExpression = + | { + type: 'binary_expr'; + operator: string; + left: SqlExpression; + right: SqlExpression; + } + | { + type: 'binary_expr'; + operator: string; + left: SqlColumnRef; + right: SqlValue; + }; + +export type SqlValue = SqlStringValue | SqlNullValue | SqlNumberValue; + +export type SqlExpression = SqlBinaryExpression | SqlUnaryExpression; + +export interface SqlSelectQuery extends NodeSqlParser.Select { + where: SqlExpression | null; +} diff --git a/yarn.lock b/yarn.lock index 3a70b3e..84a25dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3451,6 +3451,11 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +big-integer@^1.6.48: + version "1.6.51" + resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.51.tgz#0df92a5d9880560d3ff2d5fd20245c889d130686" + integrity sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg== + big.js@^5.2.2: version "5.2.2" resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" @@ -6912,6 +6917,13 @@ node-releases@^2.0.6: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.6.tgz#8a7088c63a55e493845683ebf3c828d8c51c5503" integrity sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg== +node-sql-parser@^4.6.6: + version "4.6.6" + resolved "https://registry.yarnpkg.com/node-sql-parser/-/node-sql-parser-4.6.6.tgz#910fcd4ba0132d9a5a8c312637313acc2b43b24a" + integrity sha512-zpash5xnRY6+0C9HFru32iRJV1LTkwtrVpO90i385tYVF6efyXK/B3Nsq/15Fuv2utxrqHNjKtL55OHb8sl+eQ== + dependencies: + big-integer "^1.6.48" + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"