Skip to content

Commit

Permalink
fix #648
Browse files Browse the repository at this point in the history
  • Loading branch information
@Slach
Slach committed Oct 27, 2024
1 parent 8b437f1 commit e982467
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 51 deletions.
73 changes: 59 additions & 14 deletions src/datasource/scanner/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ const tokenRe = [
const tabSize = ' '; // 4 spaces
const newLine = '\n';

export default class Scanner {
class Scanner {
tree: any;
rootToken: any;
token: any;
Expand Down Expand Up @@ -173,7 +173,6 @@ export default class Scanner {
}

_next() {

if (this._s.length === 0) {
return false;
}
Expand Down Expand Up @@ -219,7 +218,7 @@ export default class Scanner {
isExpectedNext(): boolean {
let v = this.expectedNext;
this.expectedNext = false;
return v as boolean
return v as boolean;
}

appendToken(argument): string {
Expand All @@ -242,7 +241,7 @@ export default class Scanner {
argument += this.appendToken(argument);
continue;
}
if (!isClosured(argument)) {
if (!Scanner.isClosured(argument)) {
argument += this.appendToken(argument);
continue;
}
Expand All @@ -254,7 +253,7 @@ export default class Scanner {
continue;
}

if (this.token === ',' && isClosured(argument)) {
if (this.token === ',' && Scanner.isClosured(argument)) {
this.push(argument);
argument = '';
if (this.rootToken === 'where') {
Expand Down Expand Up @@ -299,7 +298,7 @@ export default class Scanner {
}

if (isIn(this.token)) {
argument += ' ' +this.token;
argument += ' ' + this.token;
if (!this.next()) {
throw 'wrong in signature for `' + argument + '` at [' + this._s + ']';
}
Expand Down Expand Up @@ -329,7 +328,7 @@ export default class Scanner {
}

if (isCond(this.token) && (this.rootToken === 'where' || this.rootToken === 'prewhere')) {
if (isClosured(argument)) {
if (Scanner.isClosured(argument)) {
this.push(argument);
argument = this.token;
} else {
Expand Down Expand Up @@ -377,7 +376,6 @@ export default class Scanner {
}

argument += this.appendToken(argument);

}

if (argument !== '') {
Expand Down Expand Up @@ -488,9 +486,59 @@ export default class Scanner {
}

static AddMetadata(query) {
return "/* grafana dashboard=$__dashboard, user=$__user */\n" + query
}
return '/* grafana dashboard=$__dashboard, user=$__user */\n' + query;
}

static isClosured(str) {
const stack: string[] = [];
let isInQuote = false;
let quoteType = null;

const openBrackets = {
'(': ')',
'[': ']',
'{': '}',
};

const closeBrackets = {
')': '(',
']': '[',
'}': '{',
};

for (let i = 0; i < str.length; i++) {
const char = str[i];

// Handle quotes
if ((char === "'" || char === '"' || char === '`') && (i === 0 || str[i - 1] !== '\\')) {
if (!isInQuote) {
isInQuote = true;
quoteType = char;
} else if (char === quoteType) {
isInQuote = false;
quoteType = null;
}
continue;
}

// Skip characters inside quotes
if (isInQuote) {
continue;
}

// Handle brackets
if (char in openBrackets) {
stack.push(char);
} else if (char in closeBrackets) {
const lastOpen = stack.pop();
if (lastOpen !== closeBrackets[char]) {
return false;
}
}
}

return stack.length === 0;
}
}
const isSkipSpace = (token: string) => skipSpaceOnlyRe.test(token);
const isCond = (token: string) => condOnlyRe.test(token);
Expand Down Expand Up @@ -537,9 +585,6 @@ function isSet(obj, prop) {
return obj.hasOwnProperty(prop) && !isEmpty(obj[prop]);
}

function isClosured(argument) {
return (argument.match(/\(/g) || []).length === (argument.match(/\)/g) || []).length;
}

function betweenBraces(query) {
let openBraces = 1,
Expand Down Expand Up @@ -713,4 +758,4 @@ function print(AST, tab = '') {
}



export default Scanner;
129 changes: 92 additions & 37 deletions src/spec/scanner_specs.jest.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Scanner from '../datasource/scanner/scanner';

describe('scanner:', () => {
describe('Scanner:', () => {
describe('AST case 1', () => {
let query =
'SELECT EventDate, col1, col2, toUInt32(col1 > 0 ? col2/col1*10000 : 0)/100 AS percent ' +
Expand Down Expand Up @@ -648,30 +648,33 @@ describe('scanner:', () => {
const scanner = new Scanner(query);

let expectedAST = {
root: [],
'$columns': [ 'service_name', 'count() c' ],
select: [],
from: [ '$table' ],
where: [ "service_name IN ['mysql', 'postgresql'] AND $timeFilter" ]
}
;

root: [],
$columns: ['service_name', 'count() c'],
select: [],
from: ['$table'],
where: ["service_name IN ['mysql', 'postgresql'] AND $timeFilter"],
};
it('expects equality', () => {
expect(scanner.toAST()).toEqual(expectedAST);
});
});
/* https://github.com/Altinity/clickhouse-grafana/issues/386 */
describe('AST case 24 $rateColumnsAggregated', () => {
let query =
'/* comment */ $rateColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) '+
" FROM traffic WHERE datacenter = 'dc1' HAVING rx_bytes > $interval",
'/* comment */ $rateColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) ' +
" FROM traffic WHERE datacenter = 'dc1' HAVING rx_bytes > $interval",
scanner = new Scanner(query);

let expectedAST = {
root: [
"/* comment */\n"
root: ['/* comment */\n'],
$rateColumnsAggregated: [
'datacenter',
'concat(datacenter, interface) AS dc_interface',
'sum',
'tx_bytes * 1024 AS tx_kbytes',
'sum',
'max(rx_bytes) AS rx_bytes',
],
$rateColumnsAggregated: ["datacenter", "concat(datacenter, interface) AS dc_interface", "sum", "tx_bytes * 1024 AS tx_kbytes", "sum", "max(rx_bytes) AS rx_bytes",],
select: [],
from: ['traffic'],
where: ["datacenter = 'dc1'"],
Expand All @@ -686,15 +689,20 @@ describe('scanner:', () => {
/* https://github.com/Altinity/clickhouse-grafana/issues/386 */
describe('AST case 25 $perSecondColumnsAggregated', () => {
let query =
'/* comment */ $perSecondColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) '+
'/* comment */ $perSecondColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) ' +
" FROM traffic WHERE datacenter = 'dc1' HAVING rx_bytes > $interval",
scanner = new Scanner(query);

let expectedAST = {
root: [
"/* comment */\n"
root: ['/* comment */\n'],
$perSecondColumnsAggregated: [
'datacenter',
'concat(datacenter, interface) AS dc_interface',
'sum',
'tx_bytes * 1024 AS tx_kbytes',
'sum',
'max(rx_bytes) AS rx_bytes',
],
$perSecondColumnsAggregated: ["datacenter", "concat(datacenter, interface) AS dc_interface", "sum", "tx_bytes * 1024 AS tx_kbytes", "sum", "max(rx_bytes) AS rx_bytes",],
select: [],
from: ['traffic'],
where: ["datacenter = 'dc1'"],
Expand All @@ -705,19 +713,24 @@ describe('scanner:', () => {
expect(scanner.toAST()).toEqual(expectedAST);
});
});

/* https://github.com/Altinity/clickhouse-grafana/issues/386 */
describe('AST case 26 $increaseColumnsAggregated', () => {
let query =
'/* comment */ $increaseColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) '+
'/* comment */ $increaseColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) ' +
" FROM traffic WHERE datacenter = 'dc1' HAVING rx_bytes > $interval",
scanner = new Scanner(query);

let expectedAST = {
root: [
"/* comment */\n"
root: ['/* comment */\n'],
$increaseColumnsAggregated: [
'datacenter',
'concat(datacenter, interface) AS dc_interface',
'sum',
'tx_bytes * 1024 AS tx_kbytes',
'sum',
'max(rx_bytes) AS rx_bytes',
],
$increaseColumnsAggregated: ["datacenter", "concat(datacenter, interface) AS dc_interface", "sum", "tx_bytes * 1024 AS tx_kbytes", "sum", "max(rx_bytes) AS rx_bytes",],
select: [],
from: ['traffic'],
where: ["datacenter = 'dc1'"],
Expand All @@ -732,15 +745,20 @@ describe('scanner:', () => {
/* https://github.com/Altinity/clickhouse-grafana/issues/386 */
describe('AST case 27 $deltaColumnsAggregated', () => {
let query =
'/* comment */ $deltaColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) '+
'/* comment */ $deltaColumnsAggregated(datacenter, concat(datacenter,interface) AS dc_interface, sum, tx_bytes * 1024 AS tx_kbytes, sum, max(rx_bytes) AS rx_bytes) ' +
" FROM traffic WHERE datacenter = 'dc1' HAVING rx_bytes > $interval",
scanner = new Scanner(query);

let expectedAST = {
root: [
"/* comment */\n"
root: ['/* comment */\n'],
$deltaColumnsAggregated: [
'datacenter',
'concat(datacenter, interface) AS dc_interface',
'sum',
'tx_bytes * 1024 AS tx_kbytes',
'sum',
'max(rx_bytes) AS rx_bytes',
],
$deltaColumnsAggregated: ["datacenter", "concat(datacenter, interface) AS dc_interface", "sum", "tx_bytes * 1024 AS tx_kbytes", "sum", "max(rx_bytes) AS rx_bytes",],
select: [],
from: ['traffic'],
where: ["datacenter = 'dc1'"],
Expand All @@ -754,20 +772,21 @@ describe('scanner:', () => {

/* https://github.com/Altinity/clickhouse-grafana/issues/409 */
describe('AST case 28 $columns + ORDER BY WITH FILL', () => {
let query = "$columns(\n"+
" service_name, \n"+
" sum(agg_value) as value\n"+
")\n"+
"FROM $table\n" +
"WHERE service_name='mysql'\n" +
"GROUP BY t, service_name\n" +
"HAVING value>100\n"+
"ORDER BY t, service_name WITH FILL 60000",
let query =
'$columns(\n' +
' service_name, \n' +
' sum(agg_value) as value\n' +
')\n' +
'FROM $table\n' +
"WHERE service_name='mysql'\n" +
'GROUP BY t, service_name\n' +
'HAVING value>100\n' +
'ORDER BY t, service_name WITH FILL 60000',
scanner = new Scanner(query);

let expectedAST = {
root: [],
$columns: ["service_name", "sum(agg_value) as value", ],
$columns: ['service_name', 'sum(agg_value) as value'],
select: [],
from: ['$table'],
where: ["service_name = 'mysql'"],
Expand All @@ -780,5 +799,41 @@ describe('scanner:', () => {
expect(scanner.toAST()).toEqual(expectedAST);
});
});
});

// https://github.com/Altinity/clickhouse-grafana/issues/648
describe('Scanner.isClosured: ', () => {
test('handles simple brackets', () => {
expect(Scanner.isClosured('(test)')).toBe(true);
expect(Scanner.isClosured('[test]')).toBe(true);
expect(Scanner.isClosured('{test}')).toBe(true);
});

test('handles nested brackets', () => {
expect(Scanner.isClosure'\'(not a bracket)\''Be(true);
expect(Scanner.isClosured('({[test}])')).toBe(false);
});

test('handles quotes correctly', () => {
expect(Scanner.isClosured("'(not a bracket)'")); // Ignores brackets in quotes
expect(Scanner.isClosured('"[also not a bra'\'\'(this is a real bracket)\''ect(Scanner.isClosured('`{template literal}`''\\\'(this is a bracket after escaped quotes)' quotes', () => {
expect(Scanner.isClosured("''(this is a real bracket)'")).toBe(true);
ex'(\'(\'+test)'.isClosured("\\'(this is a bracket after escaped quotes)")).toBe(true);
});

test('handles provided test'(\'(\'+test+\']]\')' expect(Scanner.isClosured("('('+test)")).'\'(\'+test ]' expect(Scanner.isClosured('["("+test+"]]"]'][\'(\'+test]'e);
expect(Scanner.isClosured("('('+test+']]')")).toBe(true);
expect(Scanner.isClosured("'('+test ]")).toBe(false);
expect(Scanner.isClosured("]['('+test]")).toBe(false);
});

test('handles empty input', () => {
expect(Scanner.isClosured('')).toBe(true);
});

test('handles unmatched brackets', () => {
expect(Scanner.isClosured('(((')).toBe(false);
expect(Scanner.isClosured(')))')).toBe(false);
expect(Scanner.isClosured('((())')).toBe(false);
});
});

0 comments on commit e982467

Please sign in to comment.