From 60331562476942d15e4c3ffe68ba4a8833d98158 Mon Sep 17 00:00:00 2001 From: yakkomajuri Date: Tue, 18 May 2021 12:39:48 -0300 Subject: [PATCH 01/11] Add retry queue, + refactor --- .gitignore | 2 +- index.js | 97 -------- index.ts | 195 ++++++++++++++++ package.json | 13 ++ plugin.json | 23 +- tsconfig.json | 15 ++ yarn.lock | 611 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 857 insertions(+), 99 deletions(-) delete mode 100644 index.js create mode 100644 index.ts create mode 100644 package.json create mode 100644 tsconfig.json create mode 100644 yarn.lock diff --git a/.gitignore b/.gitignore index c6ef218..c564040 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .idea - +node_modules/ \ No newline at end of file diff --git a/index.js b/index.js deleted file mode 100644 index 5a536cc..0000000 --- a/index.js +++ /dev/null @@ -1,97 +0,0 @@ -async function setupPlugin({ global, attachments, config }) { - if (!attachments.googleCloudKeyJson) { - throw new Error('JSON config not provided!') - } - if (!config.datasetId) { - throw new Error('Dataset ID not provided!') - } - if (!config.tableId) { - throw new Error('Table ID not provided!') - } - - const credentials = JSON.parse(attachments.googleCloudKeyJson.contents.toString()) - global.bigQueryClient = new google.cloud.bigquery.BigQuery({ - projectId: credentials['project_id'], - credentials, - }) - global.bigQueryTable = global.bigQueryClient.dataset(config.datasetId).table(config.tableId) - - try { - // check if the table exists - await global.bigQueryTable.get() - } catch (error) { - // some other error? abort! - if (!error.message.includes('Not found')) { - throw new Error(error) - } - console.log(`Creating BigQuery Table - ${config.datasetId}:${config.tableId}`) - - const schema = [ - { name: 'uuid', type: 'STRING' }, - { name: 'event', type: 'STRING' }, - { name: 'properties', type: 'STRING' }, - { name: 'elements', type: 'STRING' }, - { name: 'set', type: 'STRING' }, - { name: 'set_once', type: 'STRING' }, - { name: 'distinct_id', type: 'STRING' }, - { name: 'team_id', type: 'INT64' }, - { name: 'ip', type: 'STRING' }, - { name: 'site_url', type: 'STRING' }, - { name: 'timestamp', type: 'TIMESTAMP' }, - ] - - try { - const [table] = await global.bigQueryClient - .dataset(config.datasetId) - .createTable(config.tableId, { schema }) - } catch (error) { - // a different worker already created the table - if (!error.message.includes('Already Exists')) { - throw new Error() - } - } - } -} - -async function processEventBatch(batch, { config, global }) { - if (!global.bigQueryTable) { - throw new Error('No BigQuery client initialized!') - } - - const rows = batch.map((oneEvent) => { - const { event, properties, $set, $set_once, distinct_id, team_id, site_url, now, sent_at, uuid, ..._discard } = oneEvent - const ip = properties?.['$ip'] || oneEvent.ip - const timestamp = oneEvent.timestamp || oneEvent.data?.timestamp || properties?.timestamp || now || sent_at - let ingestedProperties = properties - let elements = [] - - // only move prop to elements for the $autocapture action - if (event === '$autocapture' && properties['$elements']) { - const { $elements, ...props } = properties - ingestedProperties = props - elements = $elements - } - - return { - uuid, - event, - properties: JSON.stringify(ingestedProperties || {}), - elements: JSON.stringify(elements || {}), - set: JSON.stringify($set || {}), - set_once: JSON.stringify($set_once || {}), - distinct_id, - team_id, - ip, - site_url, - timestamp: timestamp ? global.bigQueryClient.timestamp(timestamp) : null, - } - }) - - try { - await global.bigQueryTable.insert(rows) - } catch (error) { - throw new Error(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) - } - - return batch -} \ No newline at end of file diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..55ca690 --- /dev/null +++ b/index.ts @@ -0,0 +1,195 @@ +import { createBuffer } from '@posthog/plugin-contrib' +import { Plugin, PluginMeta, PluginEvent, PluginJobs } from '@posthog/plugin-scaffold' +import { BigQuery, Table } from '@google-cloud/bigquery' + +type BigQueryMeta = PluginMeta<{ + global: { + buffer: ReturnType + eventsToIgnore: Set + retryQueue: RetryQueue + bigQueryClient: BigQuery + bigQueryTable: Table + } + config: { + datasetId: string + tableId: string + uploadMinutes: string + uploadMegabytes: string + eventsToIgnore: string + } +}> +type BigQueryPlugin = Plugin + +interface UploadJobPayload { + batch: PluginEvent[] + batchId: number +} + +class UploadError extends Error { } + +class RetryQueue { + baseInterval: number + meta: BigQueryMeta + requestRetriesMap: Map + + constructor(meta: BigQueryMeta) { + this.baseInterval = 3000 // ms + this.meta = meta + this.requestRetriesMap = new Map() + } + + async enqueue(batch: PluginEvent[], id: number) { + const { jobs } = this.meta + let retriesPerformedSoFar = 0 + if (!this.requestRetriesMap.has(id)) { + this.requestRetriesMap.set(id, 0) + } else { + retriesPerformedSoFar = this.requestRetriesMap.get(id)! + if (retriesPerformedSoFar === 15) { + this.requestRetriesMap.delete(id) + return + } + this.requestRetriesMap.set(id, retriesPerformedSoFar + 1) + } + + const nextRetryMs = 2 ** retriesPerformedSoFar * this.baseInterval + console.log(`Enqueued batch ${id} for retry in ${nextRetryMs}ms`) + + await jobs.uploadBatchToBigQuery({ batch, batchId: id }).runIn(nextRetryMs, 'milliseconds') + } +} + + +export const jobs: PluginJobs = { + uploadBatchToBigQuery: async (payload: UploadJobPayload, meta: BigQueryMeta) => { + const { global } = meta + try { + await sendBatchToBigQuery(payload.batch, meta) + } catch (err) { + console.error(err) + global.retryQueue.enqueue(payload.batch, payload.batchId) + } + }, +} + + +export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { + const { global, attachments, config, jobs } = meta + if (!attachments.googleCloudKeyJson) { + throw new Error('JSON config not provided!') + } + if (!config.datasetId) { + throw new Error('Dataset ID not provided!') + } + if (!config.tableId) { + throw new Error('Table ID not provided!') + } + + const credentials = JSON.parse(attachments.googleCloudKeyJson.contents.toString()) + const uploadMegabytes = Math.max(1, Math.min(parseInt(config.uploadMegabytes) || 1, 100)) + const uploadMinutes = Math.max(1, Math.min(parseInt(config.uploadMinutes) || 1, 60)) + + global.bigQueryClient = new BigQuery({ + projectId: credentials['project_id'], + credentials, + }) + global.bigQueryTable = global.bigQueryClient.dataset(config.datasetId).table(config.tableId) + + global.buffer = createBuffer({ + limit: uploadMegabytes * 1024 * 1024, + timeoutSeconds: uploadMinutes * 60, + onFlush: async (batch) => { + await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000) }).runNow() + }, + }) + + global.eventsToIgnore = new Set( + config.eventsToIgnore ? config.eventsToIgnore.split(',').map((event) => event.trim()) : null + ) + + global.retryQueue = new RetryQueue(meta) + + try { + // check if the table exists + await global.bigQueryTable.get() + } catch (error) { + // some other error? abort! + if (!error.message.includes('Not found')) { + throw new Error(error) + } + console.log(`Creating BigQuery Table - ${config.datasetId}:${config.tableId}`) + + const schema = [ + { name: 'uuid', type: 'STRING' }, + { name: 'event', type: 'STRING' }, + { name: 'properties', type: 'STRING' }, + { name: 'elements', type: 'STRING' }, + { name: 'set', type: 'STRING' }, + { name: 'set_once', type: 'STRING' }, + { name: 'distinct_id', type: 'STRING' }, + { name: 'team_id', type: 'INT64' }, + { name: 'ip', type: 'STRING' }, + { name: 'site_url', type: 'STRING' }, + { name: 'timestamp', type: 'TIMESTAMP' }, + ] + + try { + await global.bigQueryClient + .dataset(config.datasetId) + .createTable(config.tableId, { schema }) + } catch (error) { + // a different worker already created the table + if (!error.message.includes('Already Exists')) { + throw new Error() + } + } + } +} + +export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { + if (!global.bigQueryTable) { + throw new Error('No BigQuery client initialized!') + } + + + const { event: eventName, properties, $set, $set_once, distinct_id, team_id, site_url, now, sent_at, uuid, ..._discard } = event + const ip = properties?.['$ip'] || event.ip + const timestamp = event.timestamp || properties?.timestamp || now || sent_at + let ingestedProperties = properties + let elements = [] + + // only move prop to elements for the $autocapture action + if (eventName === '$autocapture' && properties && '$elements' in properties) { + const { $elements, ...props } = properties + ingestedProperties = props + elements = $elements + } + + const parsedEvent = { + uuid, + eventName, + properties: JSON.stringify(ingestedProperties || {}), + elements: JSON.stringify(elements || {}), + set: JSON.stringify($set || {}), + set_once: JSON.stringify($set_once || {}), + distinct_id, + team_id, + ip, + site_url, + timestamp: timestamp ? global.bigQueryClient.timestamp(timestamp) : null, + } + + if (!global.eventsToIgnore.has(eventName)) { + global.buffer.add(parsedEvent) + } +} + + +export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: BigQueryMeta) { + console.log(`Uploading ${rows.length} event ${rows.length > 1 ? 'rows' : 'row'} to BigQuery`) + try { + await global.bigQueryTable.insert(rows) + } catch (error) { + throw new UploadError(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) + } +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..d63e218 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "@posthog/s3-export-plugin", + "private": true, + "version": "0.0.1", + "description": "Export PostHog events to Amazon S3 on ingestion.", + "devDependencies": { + "@posthog/plugin-contrib": "^0.0.3", + "@posthog/plugin-scaffold": "^0.7.1", + "@types/generic-pool": "^3.1.9", + "generic-pool": "^3.7.8", + "@google-cloud/bigquery": "^5.6.0" + } +} diff --git a/plugin.json b/plugin.json index 77dd872..dfb4100 100644 --- a/plugin.json +++ b/plugin.json @@ -2,7 +2,7 @@ "name": "BigQuery Export", "url": "https://github.com/PostHog/bigquery-plugin", "description": "Sends events to a BigQuery database on ingestion.", - "main": "index.js", + "main": "index.ts", "config": [ { "key": "googleCloudKeyJson", @@ -23,6 +23,27 @@ "name": "Table ID", "type": "string", "required": true + }, + { + "key": "eventsToIgnore", + "name": "Events to ignore", + "type": "string", + "default": "$feature_flag_called", + "hint": "Comma separated list of events to ignore" + }, + { + "key": "uploadMinutes", + "name": "Upload at most every X minutes", + "type": "string", + "default": "1", + "hint": "If there are events to upload and this many minutes has passed since the last upload, send the events to S3. The value must be between 1 and 60 minutes." + }, + { + "key": "uploadMegabytes", + "name": "Maximum upload size in megabytes", + "type": "string", + "default": "1", + "hint": "Always keep the uploaded files below this size limit, uploading more frequently than the time limit if needed. Events are kept in memory until then, so make sure your server has enough of it. The value must be between 1 and 100 MB." } ] } diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8c2108e --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2018", + "lib": ["ES2019"], + "module": "ES2015", + "moduleResolution": "Node", + "strict": true, + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noUnusedParameters": true, + "esModuleInterop": true, + "noEmit": true + }, + "exclude": ["**.test.ts"] +} diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..ef459d5 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,611 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@google-cloud/bigquery@^5.6.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@google-cloud/bigquery/-/bigquery-5.6.0.tgz#1f45b23eec328bafe254c193f0f34386a4fe3e7c" + integrity sha512-uWeDCwI33L+4bx6xrVuDqGwgIqafd3Bfr0UfGAZzLYfiEStZqf8H8c8G8A/6WpUXWA3cNlYpd+KfMCcXmVg9oA== + dependencies: + "@google-cloud/common" "^3.1.0" + "@google-cloud/paginator" "^3.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + big.js "^6.0.0" + duplexify "^4.0.0" + extend "^3.0.2" + is "^3.3.0" + p-event "^4.1.0" + stream-events "^1.0.5" + uuid "^8.0.0" + +"@google-cloud/common@^3.1.0": + version "3.6.0" + resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b" + integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q== + dependencies: + "@google-cloud/projectify" "^2.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + duplexify "^4.1.1" + ent "^2.2.0" + extend "^3.0.2" + google-auth-library "^7.0.2" + retry-request "^4.1.1" + teeny-request "^7.0.0" + +"@google-cloud/paginator@^3.0.0": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-3.0.5.tgz#9d6b96c421a89bd560c1bc2c197c7611ef21db6c" + integrity sha512-N4Uk4BT1YuskfRhKXBs0n9Lg2YTROZc6IMpkO/8DIHODtm5s3xY8K5vVBo23v/2XulY3azwITQlYWgT4GdLsUw== + dependencies: + arrify "^2.0.0" + extend "^3.0.2" + +"@google-cloud/projectify@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-2.0.1.tgz#13350ee609346435c795bbfe133a08dfeab78d65" + integrity sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ== + +"@google-cloud/promisify@^2.0.0": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.3.tgz#f934b5cdc939e3c7039ff62b9caaf59a9d89e3a8" + integrity sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw== + +"@maxmind/geoip2-node@^2.3.1": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@maxmind/geoip2-node/-/geoip2-node-2.3.2.tgz#a0aaf8452693491e815021fe16e692959490ba6d" + integrity sha512-a1TSZt3uWe7yrgE2WMdTItK6XuG2Aj125cAXA+JuT2nomv3oqIK8XXg9iJVpzNXAYRV72XO0+zY+3HluOGgF3w== + dependencies: + camelcase-keys "^6.0.1" + ip6addr "^0.2.3" + lodash.set "^4.3.2" + maxmind "^4.2.0" + +"@posthog/plugin-contrib@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@posthog/plugin-contrib/-/plugin-contrib-0.0.3.tgz#d0772c6dd9ec9944ebee9dc475e1e781256b0b5f" + integrity sha512-0HrE8AuPv3OLZA93RrJDbljn9u5D/wmiIkBCeckU3LL67LNozDIJgKsY4Td91zgc+b4Rlx/X0MJNp2l6BHbQqg== + +"@posthog/plugin-scaffold@^0.7.1": + version "0.7.1" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.7.1.tgz#801dce0d55e77851fc054bd8a6284de78b90f0a3" + integrity sha512-QkMkNpyGWLen6FCmPjpHmUsalHa8e5/6eenUS/vG4SKWMORod0VDVlDYnInUXzS+HYhIjKSlgprwAyCp+2KWfQ== + dependencies: + "@maxmind/geoip2-node" "^2.3.1" + +"@tootallnate/once@1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" + integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== + +"@types/generic-pool@^3.1.9": + version "3.1.9" + resolved "https://registry.yarnpkg.com/@types/generic-pool/-/generic-pool-3.1.9.tgz#cc82ee0d92561fce713f8f9a7b2380eda8a89dcb" + integrity sha512-IkXMs8fhV6+E4J8EWv8iL7mLvApcLLQUH4m1Rex3KCPRqT+Xya0DDHIeGAokk/6VXe9zg8oTWyr+FGyeuimEYQ== + dependencies: + "@types/node" "*" + +"@types/node@*": + version "15.3.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-15.3.0.tgz#d6fed7d6bc6854306da3dea1af9f874b00783e26" + integrity sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ== + +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + +arrify@^2.0.0, arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +aws-sdk@^2.885.0: + version "2.908.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.908.0.tgz#7c1772919f2840d322a678b27625ef16a0047949" + integrity sha512-+UtrKOlwjFhGRRrtf3zl5iwFcAnvuh9m63gBnFj9aA+scbP4K2qOukJxPqXCBDeFPqLGH+ojmMJE/54oSlOfmQ== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + +base64-js@^1.0.2, base64-js@^1.3.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + +big.js@^6.0.0: + version "6.1.1" + resolved "https://registry.yarnpkg.com/big.js/-/big.js-6.1.1.tgz#63b35b19dc9775c94991ee5db7694880655d5537" + integrity sha512-1vObw81a8ylZO5ePrtMay0n018TcftpTA5HFKDaSuiUDBo8biRBtjIobw60OpwuvrGk+FsxKamqN4cnmj/eXdg== + +bignumber.js@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" + integrity sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA== + +buffer-equal-constant-time@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" + integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= + +buffer@4.9.2: + version "4.9.2" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" + integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + +camelcase-keys@^6.0.1: + version "6.2.2" + resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" + integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg== + dependencies: + camelcase "^5.3.1" + map-obj "^4.0.0" + quick-lru "^4.0.1" + +camelcase@^5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== + +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +debug@4, debug@^4.1.1: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + +duplexify@^4.0.0, duplexify@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.1.tgz#7027dc374f157b122a8ae08c2d3ea4d2d953aa61" + integrity sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: + version "1.0.11" + resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" + integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== + dependencies: + safe-buffer "^5.0.1" + +end-of-stream@^1.4.1: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + +ent@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" + integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= + +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + +extend@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" + integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + +fast-text-encoding@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" + integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== + +gaxios@^4.0.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.2.1.tgz#7463d3a06f56ddbffa745a242d2b4933b88b2ada" + integrity sha512-s+rTywpw6CmfB8r9TXYkpix7YFeuRjnR/AqhaJrQqsNhsAqej+IAiCc3hadzQH3gHyWth30tvYjxH8EVjQt/8Q== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.3.0" + +gcp-metadata@^4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.2.1.tgz#31849fbcf9025ef34c2297c32a89a1e7e9f2cd62" + integrity sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + +generic-pool@^3.7.8: + version "3.7.8" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.7.8.tgz#202087bf5ec5e0b3bae39842a0ef98bcd4c1e450" + integrity sha512-Pz93INFSbhjEROVbM91rurD05G+Kx8833rG+lVU57mznEBpzkc1f5/g+d511a1Yf8dbAEsm7DDA3QLytMFbiGA== + +google-auth-library@^7.0.2: + version "7.0.4" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.0.4.tgz#610cb010de71435dca47dfbe8dc7fbff23055d2c" + integrity sha512-o8irYyeijEiecTXeoEe8UKNEzV1X+uhR4b2oNdapDMZixypp0J+eHimGOyx5Joa3UAeokGngdtDLXtq9vDqG2Q== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + +google-p12-pem@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.0.3.tgz#673ac3a75d3903a87f05878f3c75e06fc151669e" + integrity sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA== + dependencies: + node-forge "^0.10.0" + +gtoken@^5.0.4: + version "5.2.1" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.2.1.tgz#4dae1fea17270f457954b4a45234bba5fc796d16" + integrity sha512-OY0BfPKe3QnMsY9MzTHTSKn+Vl2l1CcLe6BwDEQj00mbbkl5nyQ/7EUREstg4fQNZ8iYE7br4JJ7TdKeDOPWmw== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.0.3" + jws "^4.0.0" + +http-proxy-agent@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" + integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== + dependencies: + "@tootallnate/once" "1" + agent-base "6" + debug "4" + +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + +ieee754@^1.1.4: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + +inherits@^2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +ip6addr@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/ip6addr/-/ip6addr-0.2.3.tgz#660df0d27092434f0aadee025aba8337c6d7d4d4" + integrity sha512-qA9DXRAUW+lT47/i/4+Q3GHPwZjGt/atby1FH/THN6GVATA6+Pjp2nztH7k6iKeil7hzYnBwfSsxjthlJ8lJKw== + dependencies: + assert-plus "^1.0.0" + jsprim "^1.4.0" + +is-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" + integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + +is@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" + integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== + +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= + +jsprim@^1.4.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + +lodash.set@^4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" + integrity sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM= + +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +map-obj@^4.0.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.2.1.tgz#e4ea399dbc979ae735c83c863dd31bdf364277b7" + integrity sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ== + +maxmind@^4.2.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/maxmind/-/maxmind-4.3.1.tgz#b20103f19e9fc12e4d1618814380d89a00f65770" + integrity sha512-0CxAgwWIwQy4zF1/qCMOeUPleMTYgfnsuIsZ4Otzx6hzON4PCqivPiH6Kz7iWrC++KOGCbSB3nxkJMvDEdWt6g== + dependencies: + mmdb-lib "1.2.0" + tiny-lru "7.0.6" + +mmdb-lib@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/mmdb-lib/-/mmdb-lib-1.2.0.tgz#0ecd93f4942f65a2d09be0502fa9126939606727" + integrity sha512-3XYebkStxqCgWJjsmT9FCaE19Yi4Tvs2SBPKhUks3rJJh52oF1AKAd9kei+LTutud3a6RCZ0o2Um96Fn7o3zVA== + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +node-fetch@^2.3.0, node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + +node-forge@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" + integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== + +once@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +p-event@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/p-event/-/p-event-4.2.0.tgz#af4b049c8acd91ae81083ebd1e6f5cae2044c1b5" + integrity sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ== + dependencies: + p-timeout "^3.1.0" + +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= + +p-timeout@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + +quick-lru@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" + integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== + +readable-stream@^3.1.1: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + +retry-request@^4.1.1: + version "4.1.3" + resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.1.3.tgz#d5f74daf261372cff58d08b0a1979b4d7cab0fde" + integrity sha512-QnRZUpuPNgX0+D1xVxul6DbJ9slvo4Rm6iV/dn63e048MvGbUZiKySVt6Tenp04JqmchxjiLltGerOJys7kJYQ== + dependencies: + debug "^4.1.1" + +safe-buffer@^5.0.1, safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + +stream-events@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" + integrity sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg== + dependencies: + stubs "^3.0.0" + +stream-shift@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" + integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== + +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +stubs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" + integrity sha1-6NK6H6nJBXAwPAMLaQD31fiavls= + +teeny-request@^7.0.0: + version "7.0.1" + resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-7.0.1.tgz#bdd41fdffea5f8fbc0d29392cb47bec4f66b2b4c" + integrity sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw== + dependencies: + http-proxy-agent "^4.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.1" + stream-events "^1.0.5" + uuid "^8.0.0" + +tiny-lru@7.0.6: + version "7.0.6" + resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-7.0.6.tgz#b0c3cdede1e5882aa2d1ae21cb2ceccf2a331f24" + integrity sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow== + +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +util-deprecate@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + +uuid@^8.0.0: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== From 65cb8102c758bda002db0ddb232a30f1a4b7ea93 Mon Sep 17 00:00:00 2001 From: yakkomajuri Date: Tue, 18 May 2021 12:39:48 -0300 Subject: [PATCH 02/11] make it pretty --- index.ts | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/index.ts b/index.ts index 55ca690..03f3928 100644 --- a/index.ts +++ b/index.ts @@ -25,7 +25,7 @@ interface UploadJobPayload { batchId: number } -class UploadError extends Error { } +class UploadError extends Error {} class RetryQueue { baseInterval: number @@ -59,7 +59,6 @@ class RetryQueue { } } - export const jobs: PluginJobs = { uploadBatchToBigQuery: async (payload: UploadJobPayload, meta: BigQueryMeta) => { const { global } = meta @@ -70,8 +69,7 @@ export const jobs: PluginJobs = { global.retryQueue.enqueue(payload.batch, payload.batchId) } }, -} - +} export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { const { global, attachments, config, jobs } = meta @@ -97,7 +95,7 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { global.buffer = createBuffer({ limit: uploadMegabytes * 1024 * 1024, - timeoutSeconds: uploadMinutes * 60, + timeoutSeconds: uploadMinutes * 60, onFlush: async (batch) => { await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000) }).runNow() }, @@ -134,16 +132,14 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { ] try { - await global.bigQueryClient - .dataset(config.datasetId) - .createTable(config.tableId, { schema }) + await global.bigQueryClient.dataset(config.datasetId).createTable(config.tableId, { schema }) } catch (error) { // a different worker already created the table if (!error.message.includes('Already Exists')) { throw new Error() } } - } + } } export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { @@ -151,8 +147,19 @@ export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { throw new Error('No BigQuery client initialized!') } - - const { event: eventName, properties, $set, $set_once, distinct_id, team_id, site_url, now, sent_at, uuid, ..._discard } = event + const { + event: eventName, + properties, + $set, + $set_once, + distinct_id, + team_id, + site_url, + now, + sent_at, + uuid, + ..._discard + } = event const ip = properties?.['$ip'] || event.ip const timestamp = event.timestamp || properties?.timestamp || now || sent_at let ingestedProperties = properties @@ -184,7 +191,6 @@ export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { } } - export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: BigQueryMeta) { console.log(`Uploading ${rows.length} event ${rows.length > 1 ? 'rows' : 'row'} to BigQuery`) try { @@ -192,4 +198,4 @@ export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: BigQu } catch (error) { throw new UploadError(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) } -} \ No newline at end of file +} From 11aadef35d06730f73135793e7a3fd4dbd84ba67 Mon Sep 17 00:00:00 2001 From: yakkomajuri Date: Tue, 18 May 2021 14:21:17 -0300 Subject: [PATCH 03/11] update retry mechanism --- index.ts | 47 ++++++++++------------------------------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/index.ts b/index.ts index 03f3928..d48428b 100644 --- a/index.ts +++ b/index.ts @@ -6,7 +6,6 @@ type BigQueryMeta = PluginMeta<{ global: { buffer: ReturnType eventsToIgnore: Set - retryQueue: RetryQueue bigQueryClient: BigQuery bigQueryTable: Table } @@ -23,50 +22,26 @@ type BigQueryPlugin = Plugin interface UploadJobPayload { batch: PluginEvent[] batchId: number + retriesPerformedSoFar: number } class UploadError extends Error {} -class RetryQueue { - baseInterval: number - meta: BigQueryMeta - requestRetriesMap: Map - - constructor(meta: BigQueryMeta) { - this.baseInterval = 3000 // ms - this.meta = meta - this.requestRetriesMap = new Map() - } - - async enqueue(batch: PluginEvent[], id: number) { - const { jobs } = this.meta - let retriesPerformedSoFar = 0 - if (!this.requestRetriesMap.has(id)) { - this.requestRetriesMap.set(id, 0) - } else { - retriesPerformedSoFar = this.requestRetriesMap.get(id)! - if (retriesPerformedSoFar === 15) { - this.requestRetriesMap.delete(id) - return - } - this.requestRetriesMap.set(id, retriesPerformedSoFar + 1) - } - - const nextRetryMs = 2 ** retriesPerformedSoFar * this.baseInterval - console.log(`Enqueued batch ${id} for retry in ${nextRetryMs}ms`) - - await jobs.uploadBatchToBigQuery({ batch, batchId: id }).runIn(nextRetryMs, 'milliseconds') - } -} - export const jobs: PluginJobs = { uploadBatchToBigQuery: async (payload: UploadJobPayload, meta: BigQueryMeta) => { - const { global } = meta + const { jobs } = meta try { await sendBatchToBigQuery(payload.batch, meta) } catch (err) { console.error(err) - global.retryQueue.enqueue(payload.batch, payload.batchId) + if (payload.retriesPerformedSoFar < 15) { + const nextRetryMs = 2 ** payload.retriesPerformedSoFar * 3000 + console.log(`Enqueued batch ${payload.batchId} for retry in ${nextRetryMs}ms`) + + await jobs + .uploadBatchToBigQuery({ ...payload, retriesPerformedSoFar: payload.retriesPerformedSoFar + 1 }) + .runIn(nextRetryMs, 'milliseconds') + } } }, } @@ -105,8 +80,6 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { config.eventsToIgnore ? config.eventsToIgnore.split(',').map((event) => event.trim()) : null ) - global.retryQueue = new RetryQueue(meta) - try { // check if the table exists await global.bigQueryTable.get() From 2ff425625dcade7e756eab01de169fddd73d9125 Mon Sep 17 00:00:00 2001 From: yakkomajuri Date: Tue, 18 May 2021 14:38:54 -0300 Subject: [PATCH 04/11] final fix --- index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.ts b/index.ts index d48428b..aab3d4f 100644 --- a/index.ts +++ b/index.ts @@ -72,7 +72,7 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { limit: uploadMegabytes * 1024 * 1024, timeoutSeconds: uploadMinutes * 60, onFlush: async (batch) => { - await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000) }).runNow() + await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000), retriesPerformedSoFar: 0 }).runNow() }, }) From af08c116569c21a01c3506eb3dfd6ce967ba91ed Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Fri, 21 May 2021 14:00:37 -0300 Subject: [PATCH 05/11] update minimum posthog version --- plugin.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin.json b/plugin.json index dfb4100..887b20f 100644 --- a/plugin.json +++ b/plugin.json @@ -3,6 +3,7 @@ "url": "https://github.com/PostHog/bigquery-plugin", "description": "Sends events to a BigQuery database on ingestion.", "main": "index.ts", + "posthogVersion": ">= 1.25.0", "config": [ { "key": "googleCloudKeyJson", From 51b4fad0cbc1214f45a94cdb5b77f3ed4b124ec9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 22 May 2021 12:40:30 +0200 Subject: [PATCH 06/11] update types --- index.ts | 16 ++++---- package.json | 6 +-- yarn.lock | 105 +++------------------------------------------------ 3 files changed, 17 insertions(+), 110 deletions(-) diff --git a/index.ts b/index.ts index aab3d4f..50546e6 100644 --- a/index.ts +++ b/index.ts @@ -1,8 +1,8 @@ import { createBuffer } from '@posthog/plugin-contrib' -import { Plugin, PluginMeta, PluginEvent, PluginJobs } from '@posthog/plugin-scaffold' +import { Plugin, PluginMeta, PluginEvent } from '@posthog/plugin-scaffold' import { BigQuery, Table } from '@google-cloud/bigquery' -type BigQueryMeta = PluginMeta<{ +type BigQueryPlugin = Plugin<{ global: { buffer: ReturnType eventsToIgnore: Set @@ -16,8 +16,10 @@ type BigQueryMeta = PluginMeta<{ uploadMegabytes: string eventsToIgnore: string } + jobs: { + uploadBatchToBigQuery: UploadJobPayload + } }> -type BigQueryPlugin = Plugin interface UploadJobPayload { batch: PluginEvent[] @@ -27,8 +29,8 @@ interface UploadJobPayload { class UploadError extends Error {} -export const jobs: PluginJobs = { - uploadBatchToBigQuery: async (payload: UploadJobPayload, meta: BigQueryMeta) => { +export const jobs: BigQueryPlugin['jobs'] = { + uploadBatchToBigQuery: async (payload, meta) => { const { jobs } = meta try { await sendBatchToBigQuery(payload.batch, meta) @@ -115,7 +117,7 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { } } -export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { +export const onEvent: BigQueryPlugin['onEvent'] = (event, { global }) => { if (!global.bigQueryTable) { throw new Error('No BigQuery client initialized!') } @@ -164,7 +166,7 @@ export async function onEvent(event: PluginEvent, { global }: BigQueryMeta) { } } -export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: BigQueryMeta) { +export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: PluginMeta) { console.log(`Uploading ${rows.length} event ${rows.length > 1 ? 'rows' : 'row'} to BigQuery`) try { await global.bigQueryTable.insert(rows) diff --git a/package.json b/package.json index d63e218..da191ed 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "version": "0.0.1", "description": "Export PostHog events to Amazon S3 on ingestion.", "devDependencies": { + "@google-cloud/bigquery": "^5.6.0", "@posthog/plugin-contrib": "^0.0.3", - "@posthog/plugin-scaffold": "^0.7.1", + "@posthog/plugin-scaffold": "^0.9.0", "@types/generic-pool": "^3.1.9", - "generic-pool": "^3.7.8", - "@google-cloud/bigquery": "^5.6.0" + "generic-pool": "^3.7.8" } } diff --git a/yarn.lock b/yarn.lock index ef459d5..2a9b4f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -67,10 +67,10 @@ resolved "https://registry.yarnpkg.com/@posthog/plugin-contrib/-/plugin-contrib-0.0.3.tgz#d0772c6dd9ec9944ebee9dc475e1e781256b0b5f" integrity sha512-0HrE8AuPv3OLZA93RrJDbljn9u5D/wmiIkBCeckU3LL67LNozDIJgKsY4Td91zgc+b4Rlx/X0MJNp2l6BHbQqg== -"@posthog/plugin-scaffold@^0.7.1": - version "0.7.1" - resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.7.1.tgz#801dce0d55e77851fc054bd8a6284de78b90f0a3" - integrity sha512-QkMkNpyGWLen6FCmPjpHmUsalHa8e5/6eenUS/vG4SKWMORod0VDVlDYnInUXzS+HYhIjKSlgprwAyCp+2KWfQ== +"@posthog/plugin-scaffold@^0.9.0": + version "0.9.0" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.9.0.tgz#67f1eaeb526591ed6693dce2fd2eeb6a7ff69546" + integrity sha512-JvxTpIUsCzJcRqeEtQiAsigXyzG5fMqH0zEadEPsqfPZoznIzeNGCu37AyMewOJzId+zlgmRz1R7O1kqhbyPZg== dependencies: "@maxmind/geoip2-node" "^2.3.1" @@ -115,22 +115,7 @@ assert-plus@1.0.0, assert-plus@^1.0.0: resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= -aws-sdk@^2.885.0: - version "2.908.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.908.0.tgz#7c1772919f2840d322a678b27625ef16a0047949" - integrity sha512-+UtrKOlwjFhGRRrtf3zl5iwFcAnvuh9m63gBnFj9aA+scbP4K2qOukJxPqXCBDeFPqLGH+ojmMJE/54oSlOfmQ== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - -base64-js@^1.0.2, base64-js@^1.3.0: +base64-js@^1.3.0: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -150,15 +135,6 @@ buffer-equal-constant-time@1.0.1: resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= -buffer@4.9.2: - version "4.9.2" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" - integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - isarray "^1.0.0" - camelcase-keys@^6.0.1: version "6.2.2" resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" @@ -219,11 +195,6 @@ event-target-shim@^5.0.0: resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== -events@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= - extend@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -316,16 +287,6 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -ieee754@1.1.13: - version "1.1.13" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" - integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== - -ieee754@^1.1.4: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" - integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== - inherits@^2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" @@ -349,16 +310,6 @@ is@^3.3.0: resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== -isarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= - -jmespath@0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" - integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= - json-bigint@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" @@ -469,16 +420,6 @@ p-timeout@^3.1.0: dependencies: p-finally "^1.0.0" -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= - quick-lru@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" @@ -505,16 +446,6 @@ safe-buffer@^5.0.1, safe-buffer@~5.2.0: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -sax@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" - integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= - -sax@>=0.6.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== - stream-events@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" @@ -555,24 +486,11 @@ tiny-lru@7.0.6: resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-7.0.6.tgz#b0c3cdede1e5882aa2d1ae21cb2ceccf2a331f24" integrity sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow== -url@0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" - integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= - dependencies: - punycode "1.3.2" - querystring "0.2.0" - util-deprecate@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -uuid@3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== - uuid@^8.0.0: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" @@ -592,19 +510,6 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= - yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" From c44b886bf57f19705340214eca5afcf7ad343901 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 22 May 2021 12:52:32 +0200 Subject: [PATCH 07/11] convert to uploadSeconds --- index.ts | 6 +++--- plugin.json | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/index.ts b/index.ts index 50546e6..7d84cd8 100644 --- a/index.ts +++ b/index.ts @@ -12,7 +12,7 @@ type BigQueryPlugin = Plugin<{ config: { datasetId: string tableId: string - uploadMinutes: string + uploadSeconds: string uploadMegabytes: string eventsToIgnore: string } @@ -62,7 +62,7 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { const credentials = JSON.parse(attachments.googleCloudKeyJson.contents.toString()) const uploadMegabytes = Math.max(1, Math.min(parseInt(config.uploadMegabytes) || 1, 100)) - const uploadMinutes = Math.max(1, Math.min(parseInt(config.uploadMinutes) || 1, 60)) + const uploadSeconds = Math.max(1, Math.min(parseInt(config.uploadSeconds) || 30, 600)) global.bigQueryClient = new BigQuery({ projectId: credentials['project_id'], @@ -72,7 +72,7 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { global.buffer = createBuffer({ limit: uploadMegabytes * 1024 * 1024, - timeoutSeconds: uploadMinutes * 60, + timeoutSeconds: uploadSeconds, onFlush: async (batch) => { await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000), retriesPerformedSoFar: 0 }).runNow() }, diff --git a/plugin.json b/plugin.json index 887b20f..6cf7eae 100644 --- a/plugin.json +++ b/plugin.json @@ -33,18 +33,18 @@ "hint": "Comma separated list of events to ignore" }, { - "key": "uploadMinutes", - "name": "Upload at most every X minutes", + "key": "uploadSeconds", + "name": "Upload at least every X seconds", "type": "string", - "default": "1", - "hint": "If there are events to upload and this many minutes has passed since the last upload, send the events to S3. The value must be between 1 and 60 minutes." + "default": "30", + "hint": "If there are events to upload and this many seconds has passed since the last upload, then upload the queued events. The value must be between 1 and 600 seconds." }, { "key": "uploadMegabytes", "name": "Maximum upload size in megabytes", "type": "string", "default": "1", - "hint": "Always keep the uploaded files below this size limit, uploading more frequently than the time limit if needed. Events are kept in memory until then, so make sure your server has enough of it. The value must be between 1 and 100 MB." + "hint": "Always keep the uploaded files below this size limit, uploading more frequently than the time limit if needed. Events are kept in memory until then, so make sure your server has enough of it. The value must be between 1 and 10 MB." } ] } From d235c266f4361fdd604ad72f8add431361174ad6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 22 May 2021 21:53:18 +0200 Subject: [PATCH 08/11] split common abstraction --- index.ts | 207 ++++++++++++++++++++++++++++++---------------------- plugin.json | 18 ++--- 2 files changed, 129 insertions(+), 96 deletions(-) diff --git a/index.ts b/index.ts index 7d84cd8..23fdc7c 100644 --- a/index.ts +++ b/index.ts @@ -4,20 +4,23 @@ import { BigQuery, Table } from '@google-cloud/bigquery' type BigQueryPlugin = Plugin<{ global: { - buffer: ReturnType - eventsToIgnore: Set bigQueryClient: BigQuery bigQueryTable: Table + + exportEventsBuffer: ReturnType + exportEventsToIgnore: Set + exportEventsWithRetry: (payload: UploadJobPayload, meta: PluginMeta) => Promise } config: { datasetId: string tableId: string - uploadSeconds: string - uploadMegabytes: string - eventsToIgnore: string + + exportEventsBufferBytes: string + exportEventsBufferSeconds: string + exportEventsToIgnore: string } jobs: { - uploadBatchToBigQuery: UploadJobPayload + exportEventsWithRetry: UploadJobPayload } }> @@ -27,29 +30,10 @@ interface UploadJobPayload { retriesPerformedSoFar: number } -class UploadError extends Error {} - -export const jobs: BigQueryPlugin['jobs'] = { - uploadBatchToBigQuery: async (payload, meta) => { - const { jobs } = meta - try { - await sendBatchToBigQuery(payload.batch, meta) - } catch (err) { - console.error(err) - if (payload.retriesPerformedSoFar < 15) { - const nextRetryMs = 2 ** payload.retriesPerformedSoFar * 3000 - console.log(`Enqueued batch ${payload.batchId} for retry in ${nextRetryMs}ms`) - - await jobs - .uploadBatchToBigQuery({ ...payload, retriesPerformedSoFar: payload.retriesPerformedSoFar + 1 }) - .runIn(nextRetryMs, 'milliseconds') - } - } - }, -} +class RetryError extends Error {} export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { - const { global, attachments, config, jobs } = meta + const { global, attachments, config } = meta if (!attachments.googleCloudKeyJson) { throw new Error('JSON config not provided!') } @@ -61,27 +45,12 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { } const credentials = JSON.parse(attachments.googleCloudKeyJson.contents.toString()) - const uploadMegabytes = Math.max(1, Math.min(parseInt(config.uploadMegabytes) || 1, 100)) - const uploadSeconds = Math.max(1, Math.min(parseInt(config.uploadSeconds) || 30, 600)) - global.bigQueryClient = new BigQuery({ projectId: credentials['project_id'], credentials, }) global.bigQueryTable = global.bigQueryClient.dataset(config.datasetId).table(config.tableId) - global.buffer = createBuffer({ - limit: uploadMegabytes * 1024 * 1024, - timeoutSeconds: uploadSeconds, - onFlush: async (batch) => { - await jobs.uploadBatchToBigQuery({ batch, batchId: Math.floor(Math.random() * 1000000), retriesPerformedSoFar: 0 }).runNow() - }, - }) - - global.eventsToIgnore = new Set( - config.eventsToIgnore ? config.eventsToIgnore.split(',').map((event) => event.trim()) : null - ) - try { // check if the table exists await global.bigQueryTable.get() @@ -115,62 +84,126 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { } } } + + setupBufferExportCode(meta, exportEvents) } -export const onEvent: BigQueryPlugin['onEvent'] = (event, { global }) => { +export async function exportEvents(events: PluginEvent[], { global }: PluginMeta) { if (!global.bigQueryTable) { throw new Error('No BigQuery client initialized!') } + console.log(`Uploading ${events.length} event ${events.length > 1 ? 'events' : 'row'} to BigQuery`) + try { + const rows = events.map((event) => { + const { + event: eventName, + properties, + $set, + $set_once, + distinct_id, + team_id, + site_url, + now, + sent_at, + uuid, + ..._discard + } = event + const ip = properties?.['$ip'] || event.ip + const timestamp = event.timestamp || properties?.timestamp || now || sent_at + let ingestedProperties = properties + let elements = [] + + // only move prop to elements for the $autocapture action + if (eventName === '$autocapture' && properties && '$elements' in properties) { + const { $elements, ...props } = properties + ingestedProperties = props + elements = $elements + } - const { - event: eventName, - properties, - $set, - $set_once, - distinct_id, - team_id, - site_url, - now, - sent_at, - uuid, - ..._discard - } = event - const ip = properties?.['$ip'] || event.ip - const timestamp = event.timestamp || properties?.timestamp || now || sent_at - let ingestedProperties = properties - let elements = [] - - // only move prop to elements for the $autocapture action - if (eventName === '$autocapture' && properties && '$elements' in properties) { - const { $elements, ...props } = properties - ingestedProperties = props - elements = $elements + return { + uuid, + eventName, + properties: JSON.stringify(ingestedProperties || {}), + elements: JSON.stringify(elements || {}), + set: JSON.stringify($set || {}), + set_once: JSON.stringify($set_once || {}), + distinct_id, + team_id, + ip, + site_url, + timestamp: timestamp ? global.bigQueryClient.timestamp(timestamp) : null, + } + }) + await global.bigQueryTable.insert(rows) + } catch (error) { + throw new RetryError(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) } +} - const parsedEvent = { - uuid, - eventName, - properties: JSON.stringify(ingestedProperties || {}), - elements: JSON.stringify(elements || {}), - set: JSON.stringify($set || {}), - set_once: JSON.stringify($set_once || {}), - distinct_id, - team_id, - ip, - site_url, - timestamp: timestamp ? global.bigQueryClient.timestamp(timestamp) : null, - } +// What follows is code that should be abstracted away into the plugin server itself. - if (!global.eventsToIgnore.has(eventName)) { - global.buffer.add(parsedEvent) +const setupBufferExportCode = ( + meta: PluginMeta, + exportEvents: (events: PluginEvent[], meta: PluginMeta) => Promise +) => { + const uploadBytes = Math.max(1, Math.min(parseInt(meta.config.exportEventsBufferBytes) || 1024 * 1024, 100)) + const uploadSeconds = Math.max(1, Math.min(parseInt(meta.config.exportEventsBufferSeconds) || 30, 600)) + + meta.global.exportEventsToIgnore = new Set( + meta.config.exportEventsToIgnore + ? meta.config.exportEventsToIgnore.split(',').map((event) => event.trim()) + : null + ) + meta.global.exportEventsBuffer = createBuffer({ + limit: uploadBytes, + timeoutSeconds: uploadSeconds, + onFlush: async (batch) => { + const jobPayload = { + batch, + batchId: Math.floor(Math.random() * 1000000), + retriesPerformedSoFar: 0, + } + const firstThroughQueue = false // TODO: might make sense sometimes? e.g. when we are processing too many tasks already? + if (firstThroughQueue) { + await meta.jobs.exportEventsWithRetry(jobPayload).runNow() + } else { + await meta.global.exportEventsWithRetry(jobPayload, meta) + } + }, + }) + meta.global.exportEventsWithRetry = async (payload: UploadJobPayload, meta: PluginMeta) => { + const { jobs } = meta + try { + await exportEvents(payload.batch, meta) + } catch (err) { + if (err instanceof RetryError) { + if (payload.retriesPerformedSoFar < 15) { + const nextRetrySeconds = 2 ** payload.retriesPerformedSoFar * 3 + console.log(`Enqueued batch ${payload.batchId} for retry in ${Math.round(nextRetrySeconds)}s`) + + await jobs + .exportEventsWithRetry({ ...payload, retriesPerformedSoFar: payload.retriesPerformedSoFar + 1 }) + .runIn(nextRetrySeconds, 'seconds') + } else { + console.log( + `Dropped batch ${payload.batchId} after retrying ${payload.retriesPerformedSoFar} times` + ) + } + } else { + throw err + } + } } } -export async function sendBatchToBigQuery(rows: PluginEvent[], { global }: PluginMeta) { - console.log(`Uploading ${rows.length} event ${rows.length > 1 ? 'rows' : 'row'} to BigQuery`) - try { - await global.bigQueryTable.insert(rows) - } catch (error) { - throw new UploadError(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) +export const jobs: BigQueryPlugin['jobs'] = { + exportEventsWithRetry: async (payload, meta) => { + meta.global.exportEventsWithRetry(payload, meta) + }, +} + +export const onEvent: BigQueryPlugin['onEvent'] = (event, { global }) => { + if (!global.exportEventsToIgnore.has(event.event)) { + global.exportEventsBuffer.add(event) } } diff --git a/plugin.json b/plugin.json index 6cf7eae..60422f8 100644 --- a/plugin.json +++ b/plugin.json @@ -26,25 +26,25 @@ "required": true }, { - "key": "eventsToIgnore", + "key": "exportEventsToIgnore", "name": "Events to ignore", "type": "string", "default": "$feature_flag_called", "hint": "Comma separated list of events to ignore" }, { - "key": "uploadSeconds", - "name": "Upload at least every X seconds", + "key": "exportEventsBufferBytes", + "name": "Maximum upload size in bytes", "type": "string", - "default": "30", - "hint": "If there are events to upload and this many seconds has passed since the last upload, then upload the queued events. The value must be between 1 and 600 seconds." + "default": "1048576", + "hint": "Default 1MB. Upload events after buffering this many of them. BigQuery has a 250KB limit per row, so events are still sent individually after the buffer is flushed. The value must be between 1 and 10 MB." }, { - "key": "uploadMegabytes", - "name": "Maximum upload size in megabytes", + "key": "exportEventsBufferSeconds", + "name": "Export events at least every X seconds", "type": "string", - "default": "1", - "hint": "Always keep the uploaded files below this size limit, uploading more frequently than the time limit if needed. Events are kept in memory until then, so make sure your server has enough of it. The value must be between 1 and 10 MB." + "default": "30", + "hint": "Default 30 seconds. If there are events to upload and this many seconds has passed since the last upload, then upload the queued events. The value must be between 1 and 600 seconds." } ] } From 43ee7a175d0107a2e067e814abb3a33f9ccddadb Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sun, 23 May 2021 13:36:49 +0200 Subject: [PATCH 09/11] fix typos, undup function name --- index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.ts b/index.ts index 23fdc7c..0cf8182 100644 --- a/index.ts +++ b/index.ts @@ -85,10 +85,10 @@ export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { } } - setupBufferExportCode(meta, exportEvents) + setupBufferExportCode(meta, exportEventsToBigQuery) } -export async function exportEvents(events: PluginEvent[], { global }: PluginMeta) { +export async function exportEventsToBigQuery(events: PluginEvent[], { global }: PluginMeta) { if (!global.bigQueryTable) { throw new Error('No BigQuery client initialized!') } @@ -122,7 +122,7 @@ export async function exportEvents(events: PluginEvent[], { global }: PluginMeta return { uuid, - eventName, + event: eventName, properties: JSON.stringify(ingestedProperties || {}), elements: JSON.stringify(elements || {}), set: JSON.stringify($set || {}), From bd677ead5659a68de8349e7be70507fd9fd8e0df Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sun, 23 May 2021 16:13:54 +0200 Subject: [PATCH 10/11] log at the end --- index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.ts b/index.ts index 0cf8182..8e2a48a 100644 --- a/index.ts +++ b/index.ts @@ -92,7 +92,6 @@ export async function exportEventsToBigQuery(events: PluginEvent[], { global }: if (!global.bigQueryTable) { throw new Error('No BigQuery client initialized!') } - console.log(`Uploading ${events.length} event ${events.length > 1 ? 'events' : 'row'} to BigQuery`) try { const rows = events.map((event) => { const { @@ -135,7 +134,9 @@ export async function exportEventsToBigQuery(events: PluginEvent[], { global }: } }) await global.bigQueryTable.insert(rows) + console.log(`Inserted ${events.length} ${events.length > 1 ? 'events' : 'event'} to BigQuery`) } catch (error) { + console.error(`Error inserting ${events.length} ${events.length > 1 ? 'events' : 'event'} into BigQuery: `, error) throw new RetryError(`Error inserting into BigQuery! ${JSON.stringify(error.errors)}`) } } From 7a33677006b50449bb966cf7ded43f11d754885f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sun, 23 May 2021 16:21:24 +0200 Subject: [PATCH 11/11] take RetryError from scaffold --- index.ts | 4 +- package.json | 4 +- yarn.lock | 129 ++++----------------------------------------------- 3 files changed, 12 insertions(+), 125 deletions(-) diff --git a/index.ts b/index.ts index 8e2a48a..9b9f09b 100644 --- a/index.ts +++ b/index.ts @@ -1,5 +1,5 @@ import { createBuffer } from '@posthog/plugin-contrib' -import { Plugin, PluginMeta, PluginEvent } from '@posthog/plugin-scaffold' +import { Plugin, PluginMeta, PluginEvent, RetryError } from '@posthog/plugin-scaffold' import { BigQuery, Table } from '@google-cloud/bigquery' type BigQueryPlugin = Plugin<{ @@ -30,8 +30,6 @@ interface UploadJobPayload { retriesPerformedSoFar: number } -class RetryError extends Error {} - export const setupPlugin: BigQueryPlugin['setupPlugin'] = async (meta) => { const { global, attachments, config } = meta if (!attachments.googleCloudKeyJson) { diff --git a/package.json b/package.json index da191ed..a7b673c 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "description": "Export PostHog events to Amazon S3 on ingestion.", "devDependencies": { "@google-cloud/bigquery": "^5.6.0", - "@posthog/plugin-contrib": "^0.0.3", - "@posthog/plugin-scaffold": "^0.9.0", + "@posthog/plugin-contrib": "^0.0.4", + "@posthog/plugin-scaffold": "^0.10.0", "@types/generic-pool": "^3.1.9", "generic-pool": "^3.7.8" } diff --git a/yarn.lock b/yarn.lock index 2a9b4f1..71024ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -52,27 +52,15 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.3.tgz#f934b5cdc939e3c7039ff62b9caaf59a9d89e3a8" integrity sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw== -"@maxmind/geoip2-node@^2.3.1": - version "2.3.2" - resolved "https://registry.yarnpkg.com/@maxmind/geoip2-node/-/geoip2-node-2.3.2.tgz#a0aaf8452693491e815021fe16e692959490ba6d" - integrity sha512-a1TSZt3uWe7yrgE2WMdTItK6XuG2Aj125cAXA+JuT2nomv3oqIK8XXg9iJVpzNXAYRV72XO0+zY+3HluOGgF3w== - dependencies: - camelcase-keys "^6.0.1" - ip6addr "^0.2.3" - lodash.set "^4.3.2" - maxmind "^4.2.0" - -"@posthog/plugin-contrib@^0.0.3": - version "0.0.3" - resolved "https://registry.yarnpkg.com/@posthog/plugin-contrib/-/plugin-contrib-0.0.3.tgz#d0772c6dd9ec9944ebee9dc475e1e781256b0b5f" - integrity sha512-0HrE8AuPv3OLZA93RrJDbljn9u5D/wmiIkBCeckU3LL67LNozDIJgKsY4Td91zgc+b4Rlx/X0MJNp2l6BHbQqg== - -"@posthog/plugin-scaffold@^0.9.0": - version "0.9.0" - resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.9.0.tgz#67f1eaeb526591ed6693dce2fd2eeb6a7ff69546" - integrity sha512-JvxTpIUsCzJcRqeEtQiAsigXyzG5fMqH0zEadEPsqfPZoznIzeNGCu37AyMewOJzId+zlgmRz1R7O1kqhbyPZg== - dependencies: - "@maxmind/geoip2-node" "^2.3.1" +"@posthog/plugin-contrib@^0.0.4": + version "0.0.4" + resolved "https://registry.yarnpkg.com/@posthog/plugin-contrib/-/plugin-contrib-0.0.4.tgz#519bf98fe60f8b2fe71e5423141c1bdaa7473cc3" + integrity sha512-okPx2mIpbpiGMgWq3j9bC6u236rPFpdnSuBzWvWbmE4V9k6vSIZ4/S5w6YFnfa9o/IaudR4LVQ9h7fzXBsV5Fw== + +"@posthog/plugin-scaffold@^0.10.0": + version "0.10.0" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.10.0.tgz#e80f57c5d3833753632607bed3ca71ebf272b12b" + integrity sha512-c8lNzQTBMJ0X3SCjcaD+mXZIx2thc+ptf8G4gbfT9YBFFD6TMaxs+/APMUab2aRJRbVwOsCGj9poxwuF4wxPpA== "@tootallnate/once@1": version "1.1.2" @@ -110,11 +98,6 @@ arrify@^2.0.0, arrify@^2.0.1: resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= - base64-js@^1.3.0: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -135,25 +118,6 @@ buffer-equal-constant-time@1.0.1: resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= -camelcase-keys@^6.0.1: - version "6.2.2" - resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" - integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg== - dependencies: - camelcase "^5.3.1" - map-obj "^4.0.0" - quick-lru "^4.0.1" - -camelcase@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" - integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== - -core-util-is@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= - debug@4, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" @@ -200,16 +164,6 @@ extend@^3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= - -extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= - fast-text-encoding@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" @@ -292,14 +246,6 @@ inherits@^2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ip6addr@^0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/ip6addr/-/ip6addr-0.2.3.tgz#660df0d27092434f0aadee025aba8337c6d7d4d4" - integrity sha512-qA9DXRAUW+lT47/i/4+Q3GHPwZjGt/atby1FH/THN6GVATA6+Pjp2nztH7k6iKeil7hzYnBwfSsxjthlJ8lJKw== - dependencies: - assert-plus "^1.0.0" - jsprim "^1.4.0" - is-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" @@ -317,21 +263,6 @@ json-bigint@^1.0.0: dependencies: bignumber.js "^9.0.0" -json-schema@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" - integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= - -jsprim@^1.4.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - jwa@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" @@ -349,11 +280,6 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -lodash.set@^4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" - integrity sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM= - lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" @@ -361,24 +287,6 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -map-obj@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.2.1.tgz#e4ea399dbc979ae735c83c863dd31bdf364277b7" - integrity sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ== - -maxmind@^4.2.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/maxmind/-/maxmind-4.3.1.tgz#b20103f19e9fc12e4d1618814380d89a00f65770" - integrity sha512-0CxAgwWIwQy4zF1/qCMOeUPleMTYgfnsuIsZ4Otzx6hzON4PCqivPiH6Kz7iWrC++KOGCbSB3nxkJMvDEdWt6g== - dependencies: - mmdb-lib "1.2.0" - tiny-lru "7.0.6" - -mmdb-lib@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/mmdb-lib/-/mmdb-lib-1.2.0.tgz#0ecd93f4942f65a2d09be0502fa9126939606727" - integrity sha512-3XYebkStxqCgWJjsmT9FCaE19Yi4Tvs2SBPKhUks3rJJh52oF1AKAd9kei+LTutud3a6RCZ0o2Um96Fn7o3zVA== - ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" @@ -420,11 +328,6 @@ p-timeout@^3.1.0: dependencies: p-finally "^1.0.0" -quick-lru@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" - integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== - readable-stream@^3.1.1: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" @@ -481,11 +384,6 @@ teeny-request@^7.0.0: stream-events "^1.0.5" uuid "^8.0.0" -tiny-lru@7.0.6: - version "7.0.6" - resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-7.0.6.tgz#b0c3cdede1e5882aa2d1ae21cb2ceccf2a331f24" - integrity sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow== - util-deprecate@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -496,15 +394,6 @@ uuid@^8.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"