diff --git a/CHANGELOG.md b/CHANGELOG.md index a9831eaef42..62929db3eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ 1. [17353](https://github.com/influxdata/influxdb/pull/17353): Make all pkg resources unique by metadata.name field 1. [17363](https://github.com/influxdata/influxdb/pull/17363): Telegraf config tokens can no longer be retrieved after creation, but new tokens can be created after a telegraf has been setup 1. [17400](https://github.com/influxdata/influxdb/pull/17400): Be able to delete bucket by name via cli +1. [17396](https://github.com/influxdata/influxdb/pull/17396): Add module to write line data to specified url, org, and bucket ### Bug Fixes diff --git a/ui/src/utils/ajax.ts b/ui/src/utils/ajax.ts index 848914cecc6..0f0167ed1c8 100644 --- a/ui/src/utils/ajax.ts +++ b/ui/src/utils/ajax.ts @@ -17,6 +17,12 @@ interface RequestParams { auth?: {username: string; password: string} } +/* + * @deprecated + * + * Use fetch instead + * @ see `runQuery` in src/shared/apis/query.ts for an example + */ async function AJAX( { url, diff --git a/ui/src/utils/lineWriter.test.ts b/ui/src/utils/lineWriter.test.ts new file mode 100644 index 00000000000..43bc7762b94 --- /dev/null +++ b/ui/src/utils/lineWriter.test.ts @@ -0,0 +1,398 @@ +import {mocked} from 'ts-jest/utils' + +window.fetch = jest.fn() +import {LineWriter, Precision} from 'src/utils/lineWriter' + +describe('creating a line from a model', () => { + const lineWriter = new LineWriter( + 'http://example.com', + 'orgid', + 'bucket', + 'token==' + ) + + it('creates a line without tags', () => { + const measurement = 'performance' + const tags = {} + const fields = {fps: 55} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fps=55 1584990314') + }) + + it('creates a line when no tags are passed in', () => { + const measurement = 'performance' + const fields = {fps: 55} + + const line = lineWriter.createLineFromModel(measurement, fields) + expect(line).toEqual(expect.stringContaining('performance fps=55')) + }) + + it('creates a line without tags with multiple fields', () => { + const measurement = 'performance' + const tags = {} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fps=49.33333,heap=48577273 1584990314') + }) + + it('creates a line with a tag', () => { + const measurement = 'performance' + const tags = {region: 'us-west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region=us-west fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('creates a line with multiple tags', () => { + const measurement = 'performance' + const tags = {region: 'us-west', status: 'good'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region=us-west,status=good fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('alphabetizes tags by key, for write optimization', () => { + const measurement = 'performance' + const tags = {region: 'us-west', environment: 'dev'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,environment=dev,region=us-west fps=49.33333,heap=48577273 1584990314' + ) + }) + + describe('replacing characters which could make the parser barf', () => { + describe('measurement', () => { + it('replaces many spaces with a single escaped space', () => { + const measurement = 'performance of things' + const tags = {region: 'us-west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance\\ of\\ things,region=us-west fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('replaces commas with escaped commas', () => { + const measurement = 'performance,art' + const tags = {region: 'us-west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance\\,art,region=us-west fps=49.33333,heap=48577273 1584990314' + ) + }) + }) + }) + describe('tag keys and values', () => { + it('replaces many spaces with a single escaped space', () => { + const measurement = 'performance' + const tags = {'region of the world': 'us west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region\\ of\\ the\\ world=us\\ west fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('replaces commas with an escaped comma', () => { + const measurement = 'performance' + const tags = {'region,of,the,world': 'us,west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region\\,of\\,the\\,world=us\\,west fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('replaces equals signs with an escaped equal sign', () => { + const measurement = 'performance' + const tags = {'region=thewo=rld': 'us=west'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region\\=thewo\\=rld=us\\=west fps=49.33333,heap=48577273 1584990314' + ) + }) + + it('replaces newlines with an escaped newline in keys, and with nothing in values', () => { + const measurement = 'performance' + const tags = {'region\nworld': 'us\nwest'} + const fields = {fps: 49.33333, heap: 48577273} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe( + 'performance,region\\ world=uswest fps=49.33333,heap=48577273 1584990314' + ) + }) + }) + + describe('field keys and values', () => { + it('replaces many spaces with a single escaped space only in keys', () => { + const measurement = 'performance' + const tags = {} + const fields = {'fp s': 49.33333} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fp\\ s=49.33333 1584990314') + }) + + it('replaces commas with an escaped comma only in keys', () => { + const measurement = 'performance' + const tags = {} + const fields = {'fp,s': 49.33333} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fp\\,s=49.33333 1584990314') + }) + + it('replaces equal signs with an escaped equal sign only in keys', () => { + const measurement = 'performance' + const tags = {} + const fields = {'fp=s': 49.33333} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fp\\=s=49.33333 1584990314') + }) + + it('replaces newlines with empty strings only in values', () => { + const measurement = 'performance' + const tags = {} + const fields = {fps: '49.\n33333'} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fps=49.33333 1584990314') + }) + + it('replaces single backslashes with double backslashes only in values', () => { + const fpsString = String.raw`turk\182` + + const measurement = 'performance' + const tags = {} + const fields = {fps: fpsString} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fps=turk\\182 1584990314') + }) + + it('replaces double quotes with escaped double quotes only in values', () => { + const measurement = 'performance' + const tags = {} + const fields = {fps: '49"."33333'} + const timestamp = 1584990314 + + const line = lineWriter.createLineFromModel( + measurement, + fields, + tags, + timestamp + ) + expect(line).toBe('performance fps=49\\".\\"33333 1584990314') + }) + }) +}) + +describe('batched writes', () => { + jest.useFakeTimers() + + afterEach(() => { + mocked(window.fetch).mockReset() + }) + + const lineWriter = new LineWriter( + 'http://example.com', + 'orgid', + 'bucket', + 'token==' + ) + + it('throttles writes to 100 lines (by default) per http POST', () => { + const batchedLines = [] + + const measurement = 'performance' + const tags = {} + const timestamp = 1585163446 + for (let i = 0; i < 100; i++) { + const line = lineWriter.createLineFromModel( + measurement, + {i: i}, + tags, + timestamp + i + ) + lineWriter.batchedWrite(line, Precision.s) + batchedLines.push(line) + } + expect(mocked(window.fetch).mock.calls.length).toBe(0) + const finalLine = lineWriter.createLineFromModel( + measurement, + {i: 100}, + tags, + timestamp + 100 + ) + lineWriter.batchedWrite(finalLine, Precision.s) + batchedLines.push(finalLine) + + expect(mocked(window.fetch).mock.calls.length).toBe(1) + const [url, requestParams] = mocked(window.fetch).mock.calls[0] + + expect(url).toBe( + `http://example.com/api/v2/write?org=orgid&bucket=bucket&precision=${ + Precision.s + }` + ) + expect(requestParams).toEqual({ + method: 'POST', + body: batchedLines.join('\n'), + headers: { + Authorization: 'Token token==', + }, + }) + }) + + it('waits 10 seconds (by default) to send an HTTP reqeust', () => { + const measurement = 'performance' + const timestamp = 1585163446 + + const line = lineWriter.createLineFromModel( + measurement, + {foo: 1}, + {}, + timestamp + ) + + lineWriter.batchedWrite(line) + + jest.runAllTimers() + + expect(mocked(window.fetch).mock.calls.length).toBe(1) + expect(window.setTimeout).toHaveBeenLastCalledWith( + expect.any(Function), + 10000 + ) + + const [url, requestParams] = mocked(window.fetch).mock.calls[0] + + expect(url).toBe( + `http://example.com/api/v2/write?org=orgid&bucket=bucket&precision=${ + Precision.s + }` + ) + expect(requestParams).toEqual({ + method: 'POST', + body: line, + headers: { + Authorization: 'Token token==', + }, + }) + }) +}) diff --git a/ui/src/utils/lineWriter.ts b/ui/src/utils/lineWriter.ts new file mode 100644 index 00000000000..eeb97092dc3 --- /dev/null +++ b/ui/src/utils/lineWriter.ts @@ -0,0 +1,174 @@ +export interface Tags { + [key: string]: string +} + +export interface Fields { + [key: string]: number | string +} + +export enum Precision { + ns = 'ns', + u = 'u', + ms = 'ms', + s = 's', + m = 'm', + h = 'h', +} + +export interface BatchOptions { + maxIntervalInSeconds: number + maxBatchedLines: number +} + +const defaultBatchOptions: BatchOptions = { + maxIntervalInSeconds: 10, + maxBatchedLines: 100, +} + +const nowInSeconds = function nowInSeconds() { + return Math.floor(Date.now() / 1000) +} + +export class LineWriter { + protected url: string + protected orgID: number | string + protected bucketName: string + protected authToken: string + protected batchOptions: BatchOptions + + protected timerID: number | null + protected batchedLines: number + protected lines: string[] + + constructor( + url: string, + orgID: number | string, + bucketName: string, + authToken: string, + batchOptions: BatchOptions = defaultBatchOptions + ) { + this.url = url + this.orgID = orgID + this.bucketName = bucketName + this.authToken = authToken + this.batchOptions = batchOptions + + this.timerID = null + this.batchedLines = 0 + this.lines = [] + } + + public createLineFromModel( + measurement: string, + fields: Fields, + tags: Tags = {}, + timestamp: number = nowInSeconds() + ): string { + let tagString = '' + Object.keys(tags) + // Sort keys for a little extra perf + // https://v2.docs.influxdata.com/v2.0/write-data/best-practices/optimize-writes/#sort-tags-by-key + .sort((a, b) => a.localeCompare(b)) + .forEach((tagKey, i, tagKeys) => { + const tagValue = tags[tagKey] + const printableTagKey = tagKey + .replace(/\s+/g, '\\ ') // replace any number of spaces with an escaped space + .replace(/,/g, '\\,') // replace commas with escaped commas + .replace(/=/g, '\\=') // replace equal signs with escaped equal signs + + const printableTagValue = tagValue + .replace(/\n/g, '') // remove newlines + .replace(/\s+/g, '\\ ') // replace any number of spaces with an escaped space + .replace(/,/g, '\\,') // replace commas with escaped commas + .replace(/=/g, '\\=') // replace equal signs with escaped equal signs + + tagString = `${tagString}${printableTagKey}=${printableTagValue}` + + // if this isn't the end of the string, append a comma + if (i < tagKeys.length - 1) { + tagString = `${tagString},` + } + }) + + let fieldString = '' + Object.keys(fields).forEach((fieldKey, i, fieldKeys) => { + const fieldValue = fields[fieldKey] + + const printableFieldKey = fieldKey + .replace(/\s+/g, '\\ ') // replace any number of spaces with an escaped space + .replace(/,/g, '\\,') // replace commas with escaped commas + .replace(/=/g, '\\=') // replace equal signs with escaped equal signs + + let printableFieldValue = fieldValue + if (typeof fieldValue === 'string') { + printableFieldValue = fieldValue + .replace(/\n/g, '') // remove newlines + .replace(/\\/g, '\\') // replace single backslach with an escaped backslash + .replace(/"/g, '\\"') // replace double quotes with escaped double quotes + } + + fieldString = `${fieldString}${printableFieldKey}=${printableFieldValue}` + + // if this isn't the end of the string, append a comma + if (i < fieldKeys.length - 1) { + fieldString = `${fieldString},` + } + }) + + let lineStart = measurement + .replace(/\s+/g, '\\ ') // replace any number of spaces with an escaped space + .replace(/,/g, '\\,') // replace commas with escaped commas + + if (tagString !== '') { + lineStart = `${lineStart},${tagString}` + } + + return `${lineStart} ${fieldString} ${timestamp}` + } + + public batchedWrite = (line: string, precision: Precision = Precision.s) => { + this.lines.push(line) + this.batchedLines++ + + if (this.batchedLines > this.batchOptions.maxBatchedLines) { + this.writeLine(this.lines.join('\n'), precision) + window.clearTimeout(this.timerID) + this.timerID = null + this.batchedLines = 0 + this.lines = [] + return + } + + if (this.timerID) { + return + } + + this.timerID = window.setTimeout(() => { + this.writeLine(this.lines.join('\n'), precision) + window.clearTimeout(this.timerID) + this.timerID = null + this.batchedLines = 0 + this.lines = [] + }, this.batchOptions.maxIntervalInSeconds * 1000) + } + + protected writeLine = async ( + line: string, + precision: Precision = Precision.s + ) => { + const url = `${this.url}/api/v2/write?org=${this.orgID}&bucket=${ + this.bucketName + }&precision=${precision}` + try { + return await fetch(url, { + method: 'POST', + headers: { + Authorization: `Token ${this.authToken}`, + }, + body: line, + }) + } catch (error) { + console.error(error) + } + } +}