From 6e2710c279ddcc5ef491fda7d8ed87672ee7107f Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 11:38:26 -0700 Subject: [PATCH 1/7] Proposed solution: Node datafile manager uses gzip,deflate compression for datafile requests --- packages/datafile-manager/__test__/rawTest.ts | 19 +++++++++++++++++++ packages/datafile-manager/package-lock.json | 13 +++++++++++++ packages/datafile-manager/package.json | 3 ++- packages/datafile-manager/src/nodeRequest.ts | 14 ++++++++++---- 4 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 packages/datafile-manager/__test__/rawTest.ts diff --git a/packages/datafile-manager/__test__/rawTest.ts b/packages/datafile-manager/__test__/rawTest.ts new file mode 100644 index 000000000..2bba41f72 --- /dev/null +++ b/packages/datafile-manager/__test__/rawTest.ts @@ -0,0 +1,19 @@ +import { HttpPollingDatafileManager } from '../src/index.node'; +import { setLogLevel, setLogHandler, ConsoleLogHandler } from '@optimizely/js-sdk-logging'; + +setLogLevel('debug'); +setLogHandler(new ConsoleLogHandler()); + +const manager = new HttpPollingDatafileManager({ + sdkKey: '9LCprAQyd1bs1BBXZ3nVji', + updateInterval: 5000, +}); + +manager.start(); +manager.onReady().then(() => { + console.log('ready'); +}); + +manager.on('update', () => { + console.log('update: '); +}); diff --git a/packages/datafile-manager/package-lock.json b/packages/datafile-manager/package-lock.json index 775bb6040..8accdbae9 100644 --- a/packages/datafile-manager/package-lock.json +++ b/packages/datafile-manager/package-lock.json @@ -1357,6 +1357,14 @@ "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", "dev": true }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-assign": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/deep-assign/-/deep-assign-3.0.0.tgz", @@ -4102,6 +4110,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", diff --git a/packages/datafile-manager/package.json b/packages/datafile-manager/package.json index 7bbc26810..9d6223c0e 100644 --- a/packages/datafile-manager/package.json +++ b/packages/datafile-manager/package.json @@ -44,7 +44,8 @@ }, "dependencies": { "@optimizely/js-sdk-logging": "^0.1.0", - "@optimizely/js-sdk-utils": "^0.2.0" + "@optimizely/js-sdk-utils": "^0.2.0", + "decompress-response": "^4.2.1" }, "peerDependencies": { "@react-native-community/async-storage": "^1.2.0" diff --git a/packages/datafile-manager/src/nodeRequest.ts b/packages/datafile-manager/src/nodeRequest.ts index be5df9c34..842962743 100644 --- a/packages/datafile-manager/src/nodeRequest.ts +++ b/packages/datafile-manager/src/nodeRequest.ts @@ -19,6 +19,7 @@ import https from 'https'; import url from 'url'; import { Headers, AbortableRequest, Response } from './http'; import { REQUEST_TIMEOUT_MS } from './config'; +import decompressResponse from 'decompress-response'; // Shared signature between http.request and https.request type ClientRequestCreator = (options: http.RequestOptions) => http.ClientRequest; @@ -74,16 +75,18 @@ function getResponseFromRequest(request: http.ClientRequest): Promise return; } - incomingMessage.setEncoding('utf8'); + const response = decompressResponse(incomingMessage); + + response.setEncoding('utf8'); let responseData = ''; - incomingMessage.on('data', (chunk: string) => { + response.on('data', (chunk: string) => { if (!request.aborted) { responseData += chunk; } }); - incomingMessage.on('end', () => { + response.on('end', () => { if (request.aborted) { return; } @@ -131,7 +134,10 @@ export function makeGetRequest(reqUrl: string, headers: Headers): AbortableReque const requestOptions: http.RequestOptions = { ...getRequestOptionsFromUrl(parsedUrl), method: 'GET', - headers, + headers: { + ...headers, + 'Accept-Encoding': 'gzip,deflate', + }, }; const request = requester(requestOptions); From 3e39088bb456dccc485ab97462f33b3a686d1de9 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:04:51 -0700 Subject: [PATCH 2/7] Unit test --- .../datafile-manager/__test__/nodeRequest.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/datafile-manager/__test__/nodeRequest.spec.ts b/packages/datafile-manager/__test__/nodeRequest.spec.ts index dec6d5138..075907113 100644 --- a/packages/datafile-manager/__test__/nodeRequest.spec.ts +++ b/packages/datafile-manager/__test__/nodeRequest.spec.ts @@ -15,6 +15,7 @@ */ import nock from 'nock'; +import zlib from 'zlib'; import { makeGetRequest } from '../src/nodeRequest'; import { advanceTimersByTime } from './testUtils'; @@ -80,6 +81,20 @@ describe('nodeEnvironment', () => { scope.done(); }); + it('adds an Accept-Encoding request header and unzips a gzipped response body', async () => { + const scope = nock(host) + .matchHeader('accept-encoding', 'gzip,deflate') + .get(path) + .reply(200, () => zlib.gzipSync('{"foo":"bar"}'), { 'content-encoding': 'gzip' }); + const req = makeGetRequest(`${host}${path}`, {}); + const resp = await req.responsePromise; + expect(resp).toMatchObject({ + statusCode: 200, + body: '{"foo":"bar"}', + }); + scope.done(); + }); + it('includes headers from the response in the eventual response in the return value', async () => { const scope = nock(host) .get(path) From e8b305c97c4dcd1c34ca40aa8a22502ab277e219 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:05:23 -0700 Subject: [PATCH 3/7] Remove manual test --- packages/datafile-manager/__test__/rawTest.ts | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 packages/datafile-manager/__test__/rawTest.ts diff --git a/packages/datafile-manager/__test__/rawTest.ts b/packages/datafile-manager/__test__/rawTest.ts deleted file mode 100644 index 2bba41f72..000000000 --- a/packages/datafile-manager/__test__/rawTest.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { HttpPollingDatafileManager } from '../src/index.node'; -import { setLogLevel, setLogHandler, ConsoleLogHandler } from '@optimizely/js-sdk-logging'; - -setLogLevel('debug'); -setLogHandler(new ConsoleLogHandler()); - -const manager = new HttpPollingDatafileManager({ - sdkKey: '9LCprAQyd1bs1BBXZ3nVji', - updateInterval: 5000, -}); - -manager.start(); -manager.onReady().then(() => { - console.log('ready'); -}); - -manager.on('update', () => { - console.log('update: '); -}); From cd96fd62d4a981cfb8283f61f09057ed846b700f Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:09:54 -0700 Subject: [PATCH 4/7] Use lower-cased header name --- packages/datafile-manager/src/nodeRequest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/datafile-manager/src/nodeRequest.ts b/packages/datafile-manager/src/nodeRequest.ts index 842962743..939bac536 100644 --- a/packages/datafile-manager/src/nodeRequest.ts +++ b/packages/datafile-manager/src/nodeRequest.ts @@ -136,7 +136,7 @@ export function makeGetRequest(reqUrl: string, headers: Headers): AbortableReque method: 'GET', headers: { ...headers, - 'Accept-Encoding': 'gzip,deflate', + 'accept-encoding': 'gzip,deflate', }, }; From 26a04c69e6533d6fe117cf733635ca1eaec8c2d2 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:22:14 -0700 Subject: [PATCH 5/7] README and CHANGELOG --- packages/datafile-manager/CHANGELOG.md | 5 +++++ packages/datafile-manager/README.md | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/packages/datafile-manager/CHANGELOG.md b/packages/datafile-manager/CHANGELOG.md index 48ee813a1..dbc9ce49a 100644 --- a/packages/datafile-manager/CHANGELOG.md +++ b/packages/datafile-manager/CHANGELOG.md @@ -9,6 +9,11 @@ Changes that have landed but are not yet released. ### Breaking Changes - Removed `StaticDatafileManager` from all top level exports +- Dropped support for Node.js version <8 + +### Fixed + +- Node datafile manager requests use gzip,deflate compression ## [0.4.0] - June 12, 2019 diff --git a/packages/datafile-manager/README.md b/packages/datafile-manager/README.md index 6da3577de..91b0be7df 100644 --- a/packages/datafile-manager/README.md +++ b/packages/datafile-manager/README.md @@ -2,6 +2,14 @@ This package provides datafile manager implementations for Node.js, browsers, and React Native. +## Requirements +In general, an ES5-compatible environment is required, as well as `Promise` (must be polyfilled if absent). + +Platform-specific minimum supported versions: + +- Node.js: `8` +- React Native: `0.61.5` + ## Installation ```sh From 03c1f1b463dedf3f91864d7dff888b9120c5b384 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:49:47 -0700 Subject: [PATCH 6/7] Update additional code in README, add additional code to root readme --- README.md | 33 +++++++++++++++++++++++++++++++ packages/optimizely-sdk/README.md | 16 +++++---------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a30d57f5d..243970373 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,36 @@ This repository is a monorepo that we manage using [Lerna](https://github.com/le ### Contributing Please see [CONTRIBUTING](CONTRIBUTING.md). + +## Credits + +First-party code (under packages/optimizely-sdk/lib/, packages/datafile-manager/lib, packages/datafile-manager/src, packages/datafile-manager/__test__, packages/event-processor/src, packages/event-processor/__tests__, packages/logging/src, packages/logging/__tests__, packages/utils/src, packages/utils/__tests__) is copyright Optimizely, Inc. and contributors, licensed under Apache 2.0. + +## Additional Code + +Prod dependencies are as follows: + +```json +{ + "json-schema@0.2.3": { + "licenses": [ + "AFLv2.1", + "BSD" + ], + "publisher": "Kris Zyp", + "repository": "https://github.com/kriszyp/json-schema" + }, + "murmurhash@0.0.2": { + "licenses": "MIT*", + "repository": "https://github.com/perezd/node-murmurhash" + }, + "uuid@3.3.2": { + "licenses": "MIT", + "repository": "https://github.com/kelektiv/node-uuid" + }, + "decompress-response@4.2.1": { + "licenses": "MIT", + "repository": "https://github.com/sindresorhus/decompress-response" + } +} +``` diff --git a/packages/optimizely-sdk/README.md b/packages/optimizely-sdk/README.md index 896069ced..9459dddc8 100644 --- a/packages/optimizely-sdk/README.md +++ b/packages/optimizely-sdk/README.md @@ -96,23 +96,17 @@ Prod dependencies are as follows: "publisher": "Kris Zyp", "repository": "https://github.com/kriszyp/json-schema" }, - "lodash@4.17.10": { - "licenses": "MIT", - "publisher": "John-David Dalton", - "repository": "https://github.com/lodash/lodash" - }, "murmurhash@0.0.2": { "licenses": "MIT*", "repository": "https://github.com/perezd/node-murmurhash" }, - "sprintf@0.1.5": { - "licenses": "BSD-3-Clause", - "publisher": "Moritz Peters", - "repository": "https://github.com/maritz/node-sprintf" - }, - "uuid@3.2.1": { + "uuid@3.3.2": { "licenses": "MIT", "repository": "https://github.com/kelektiv/node-uuid" + }, + "decompress-response@4.2.1": { + "licenses": "MIT", + "repository": "https://github.com/sindresorhus/decompress-response" } } ``` From 7b21a77bf3682eae59797244e62513525b99fdb3 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 16 Apr 2020 15:55:06 -0700 Subject: [PATCH 7/7] Fix formatting in credits --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 243970373..668ab161f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Please see [CONTRIBUTING](CONTRIBUTING.md). ## Credits -First-party code (under packages/optimizely-sdk/lib/, packages/datafile-manager/lib, packages/datafile-manager/src, packages/datafile-manager/__test__, packages/event-processor/src, packages/event-processor/__tests__, packages/logging/src, packages/logging/__tests__, packages/utils/src, packages/utils/__tests__) is copyright Optimizely, Inc. and contributors, licensed under Apache 2.0. +First-party code (under `packages/optimizely-sdk/lib/`, `packages/datafile-manager/lib`, `packages/datafile-manager/src`, `packages/datafile-manager/__test__`, `packages/event-processor/src`, `packages/event-processor/__tests__`, `packages/logging/src`, `packages/logging/__tests__`, `packages/utils/src`, `packages/utils/__tests__`) is copyright Optimizely, Inc. and contributors, licensed under Apache 2.0. ## Additional Code