From 2c4459f5c593979eb52809491409f38b83b1394e Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 17:11:07 +0300 Subject: [PATCH 01/11] test local --- src/tests/http-s3.test.ts | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index cd9d551..7bb7688 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -36,14 +36,13 @@ describe("Http/s3 interface", () => { ]); }); - it("allows reading from s3", async () => { + // Note, the github actions test uses "dforsber-duckdb-test" user in dev + it("allows reading from s3 - github actions", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); - // NOTE: The "=" sign had to be encoded to "%3D" to make this work, - // probably something that DuckDB should be doing! const result = await connection.executeIterator( - "SELECT * FROM parquet_scan('s3://amazon-reviews-pds/parquet/product_category%3DBooks/part-00000-495c48e6-96d6-4650-aa65-3c36a3516ddd.c000.snappy.parquet') LIMIT 1", + "SELECT * FROM parquet_scan('s3://amazon-reviews-pds/parquet/product_category=Books/part-00000-495c48e6-96d6-4650-aa65-3c36a3516ddd.c000.snappy.parquet') LIMIT 1", executeOptions, ); expect(result.fetchRow()).toMatchObject([ @@ -64,4 +63,29 @@ describe("Http/s3 interface", () => { 2015, ]); }); + + // TODO: merge the two s3 tests in one by using the same s3 bucket but different users + it("allows reading from s3 - local", async () => { + await connection.executeIterator(`SET s3_region='us-east-1'`); + await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); + await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); + await connection.executeIterator(`SET s3_session_token='${process.env.AWS_SESSION_TOKEN}'`); + const result = await connection.executeIterator( + "SELECT * FROM parquet_scan('s3://node-duckdb-test-bucket/alltypes_plain.parquet') LIMIT 1", + executeOptions, + ); + expect(result.fetchRow()).toMatchObject([ + 4, + true, + 0, + 0, + 0, + 0n, + 0, + 0, + Buffer.from("03/01/09"), + Buffer.from("0"), + 1235865600000, + ]); + }); }); From bee2f5fbcaab722c782ac6713ee02a6d86fbde40 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 17:13:23 +0300 Subject: [PATCH 02/11] wip --- addon/duckdb.cc | 2 +- docker-compose.yml | 3 +++ package.json | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/addon/duckdb.cc b/addon/duckdb.cc index a6f16bc..7f89b28 100644 --- a/addon/duckdb.cc +++ b/addon/duckdb.cc @@ -71,7 +71,7 @@ DuckDB::DuckDB(const Napi::CallbackInfo &info) throw Napi::Error::New(env, e.what()); } catch (...) { throw Napi::Error::New(env, - "An error occured during DuckDB initialisation"); + "An error occurred during DuckDB initialisation"); } } diff --git a/docker-compose.yml b/docker-compose.yml index 680130c..a23e302 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,9 @@ services: environment: YARN_CACHE_FOLDER: /yarn_cache GITHUB_TOKEN: $GITHUB_TOKEN + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN: $AWS_SESSION_TOKEN volumes: - .:/root/node-duckdb - ~/.npmrc:/root/.npmrc diff --git a/package.json b/package.json index 4419986..d59ab51 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "clang:check": "yarn clang-format --dry-run --Werror addon/**", "clang:fix": "yarn clang-format -i addon/**", "cleanup:binaries": "rm -rf build prebuilds duckdb", - "download-duckdb": "rm -rf duckdb && curl -L https://github.com/cwida/duckdb/archive/v0.2.8.tar.gz > duckdb.tar.gz && tar xf duckdb.tar.gz && mv duckdb-0.2.8 duckdb && rm duckdb.tar.gz", + "download-duckdb": "rm -rf duckdb && curl -L https://github.com/cwida/duckdb/archive/c07e229072595e086dc0adfac72638e8cc72de40.tar.gz > duckdb.tar.gz && tar xf duckdb.tar.gz && mv duckdb-c07e229072595e086dc0adfac72638e8cc72de40 duckdb && rm duckdb.tar.gz", "download-duckdb:master": "rm -rf duckdb && git clone https://github.com/duckdb/duckdb.git && cd duckdb && git checkout 31308be8cb5b8382920b6e39ae1d2923da682edc", "eslint:check": "eslint --ext .js,.json,.ts ./", "eslint:fix": "eslint --fix --ext .js,.json,.ts ./", From a2ec017718dd6c74abe06a078418851830c2e696 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 17:23:33 +0300 Subject: [PATCH 03/11] fix --- package.json | 3 +-- src/tests/execute-error-handling.test.ts | 15 ++++++++------- src/tests/http-s3.test.ts | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index d59ab51..76d0e8c 100644 --- a/package.json +++ b/package.json @@ -41,11 +41,10 @@ "clang:fix": "yarn clang-format -i addon/**", "cleanup:binaries": "rm -rf build prebuilds duckdb", "download-duckdb": "rm -rf duckdb && curl -L https://github.com/cwida/duckdb/archive/c07e229072595e086dc0adfac72638e8cc72de40.tar.gz > duckdb.tar.gz && tar xf duckdb.tar.gz && mv duckdb-c07e229072595e086dc0adfac72638e8cc72de40 duckdb && rm duckdb.tar.gz", - "download-duckdb:master": "rm -rf duckdb && git clone https://github.com/duckdb/duckdb.git && cd duckdb && git checkout 31308be8cb5b8382920b6e39ae1d2923da682edc", "eslint:check": "eslint --ext .js,.json,.ts ./", "eslint:fix": "eslint --fix --ext .js,.json,.ts ./", "generate-doc": "yarn build:ts && rm -rf temp etc && mkdir etc && yarn api-extractor run --local --verbose && yarn api-documenter markdown -i temp -o docs/api && ./docs/replace.sh", - "install": "prebuild-install --verbose -d -r napi || (yarn download-duckdb:master && yarn build:duckdb && yarn prebuild:current-target)", + "install": "prebuild-install --verbose -d -r napi || (yarn download-duckdb && yarn build:duckdb && yarn prebuild:current-target)", "lint:check": "yarn prettier:check && yarn eslint:check && yarn clang:check", "lint:fix": "yarn prettier:fix && yarn eslint:fix && yarn clang:fix", "prebuild:all-targets": "yarn install && yarn prebuild:linux", diff --git a/src/tests/execute-error-handling.test.ts b/src/tests/execute-error-handling.test.ts index d4d21b8..6935a92 100644 --- a/src/tests/execute-error-handling.test.ts +++ b/src/tests/execute-error-handling.test.ts @@ -1,9 +1,5 @@ import { Connection, DuckDB } from "@addon"; -const invalidQueryError = `Parser Error: syntax error at or near "an" -LINE 1: an invalid query - ^`; - describe("executeIterator method error handling", () => { let db: DuckDB; let connection: Connection; @@ -23,9 +19,14 @@ describe("executeIterator method error handling", () => { }); }); it("correctly handles an invalid query", async () => { - await expect(connection.executeIterator("an invalid query")).rejects.toMatchObject({ - message: invalidQueryError, - }); + let error: Error; + try { + await connection.executeIterator("an invalid query"); + } catch (e) { + error = e; + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + expect(error!.message).toContain("an invalid query"); }); it("correctly handles a failing query - file does not exist", async () => { await expect(connection.executeIterator("SELECT * FROM read_csv_auto('/idontexist.csv')")).rejects.toMatchObject({ diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index 7bb7688..dbcc80d 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -71,8 +71,8 @@ describe("Http/s3 interface", () => { await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); await connection.executeIterator(`SET s3_session_token='${process.env.AWS_SESSION_TOKEN}'`); const result = await connection.executeIterator( - "SELECT * FROM parquet_scan('s3://node-duckdb-test-bucket/alltypes_plain.parquet') LIMIT 1", - executeOptions, + "SELECT * FROM parquet_scan('s3://node-duckdb-test-bucket/alltypes_plain.parquet') LIMIT 1", + executeOptions, ); expect(result.fetchRow()).toMatchObject([ 4, From d416dea2759636fe5e226bada60fbe70f7e1b2db Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 17:49:40 +0300 Subject: [PATCH 04/11] dockerfile --- Dockerfile | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd54b63..98deb44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,13 @@ -FROM 059037012730.dkr.ecr.us-east-1.amazonaws.com/deepcrawl/odin-api:latest +FROM amazon/aws-lambda-nodejs:14 WORKDIR /app -RUN yum update -y && yum -y install openssl11-devel +RUN yum update -y && \ + yum -y install openssl11-devel && \ + yum install make -y && \ + yum install yum install gcc-c++ -y && \ + yum install cmake3 -y && \ + yum remove cmake -y && \ + ln -s /usr/bin/cmake3 /usr/bin/cmake && \ + npm install -g yarn +RUN curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 +RUN chmod +x /usr/local/bin/dumb-init +ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] From 2c953474ff381271408f77c2093d9c975ee0c414 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 18:03:27 +0300 Subject: [PATCH 05/11] cleanup --- docs/DEVELOPING.md | 11 ++++++++++- src/tests/http-s3.test.ts | 8 ++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index bfa9f4f..be97476 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -1,11 +1,20 @@ ## Developing -First build: +### First build: 1. `yarn install` - installs dependencies including downloading duckdb 2. `yarn build:ts` - builds typescript 3. `yarn test` - runs all tests +### Build using DuckDB sources: + +Prerequisites: + +- OpenSSL v1.1+ (on Mac you may need to specify `OPENSSL_ROOT_DIR`, e.g. ` export OPENSSL_ROOT_DIR=/usr/local/opt/openssl`) + +1. `yarn download-duckdb`: downloads the version of duckdb specified in the package.json command +2. `yarn build`: will build the DuckDB sources, the C++ addon sources and the typescript sources + Other useful scripts: - `yarn build` - build everything diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index dbcc80d..e8f835b 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -36,7 +36,9 @@ describe("Http/s3 interface", () => { ]); }); - // Note, the github actions test uses "dforsber-duckdb-test" user in dev + // Note, this test uses "dforsber-duckdb-test" user in dev + // (won't pass with your regular DC creds since they don't have permissions to the bucket + // and it also needs the session token) it("allows reading from s3 - github actions", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); @@ -65,7 +67,9 @@ describe("Http/s3 interface", () => { }); // TODO: merge the two s3 tests in one by using the same s3 bucket but different users - it("allows reading from s3 - local", async () => { + // Use DC Dev creds for this test (need to export AWS_ACCESS_KEY_ID, etc) + // Note, won't pass in github actions since that user doesn't have DC Dev perms + it.skip("allows reading from s3 - local", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); From 33377d3b218f1b8fbe9cbb73cdd55dbd391133df Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 18:05:56 +0300 Subject: [PATCH 06/11] audit --- yarn.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6dfa63a..5062505 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1319,9 +1319,9 @@ ansi-regex@^4.0.0, ansi-regex@^4.1.0: integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-styles@^3.2.0, ansi-styles@^3.2.1: version "3.2.1" @@ -1522,11 +1522,11 @@ aws4@^1.8.0: integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== axios@^0.21.1: - version "0.21.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8" - integrity sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA== + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== dependencies: - follow-redirects "^1.10.0" + follow-redirects "^1.14.0" babel-jest@^26.6.3: version "26.6.3" @@ -3518,10 +3518,10 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561" integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA== -follow-redirects@^1.10.0: - version "1.14.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.2.tgz#cecb825047c00f5e66b142f90fed4f515dec789b" - integrity sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA== +follow-redirects@^1.14.0: + version "1.14.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.6.tgz#8cfb281bbc035b3c067d6cd975b0f6ade6e855cd" + integrity "sha1-jPsoG7wDWzwGfWzZdbD2reboVc0= sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A==" for-in@^1.0.2: version "1.0.2" @@ -8137,9 +8137,9 @@ tiny-relative-date@^1.3.0: integrity sha512-MOQHpzllWxDCHHaDno30hhLfbouoYlOI8YlMNtvKe1zXbjEVhbcEovQxvZrPvtiYW630GQDoMMarCnjfyfHA+A== tmpl@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" - integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE= + version "1.0.5" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== to-fast-properties@^2.0.0: version "2.0.0" From cfb6084a637c5bc51d00d4ffa3e3014917a01b59 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 18:43:23 +0300 Subject: [PATCH 07/11] s3 test refactor --- .github/workflows/build.yml | 3 +-- docs/DEVELOPING.md | 4 ++++ package.json | 5 +++- scripts/setup-test-environment.sh | 5 ++++ src/tests/http-s3.test.ts | 39 +++++++------------------------ 5 files changed, 23 insertions(+), 33 deletions(-) create mode 100755 scripts/setup-test-environment.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cadd23f..bd2e858 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,8 +35,7 @@ jobs: run: | yarn install --frozen-lockfile yarn lint:check - yarn audit - yarn test + yarn test:ci env: NODE_AUTH_TOKEN: ${{secrets.GIT_REGISTRY_TOKEN}} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index be97476..cd74e5f 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -2,6 +2,10 @@ ### First build: +Prerequisites: + +- aws cli tools (for the s3 test) + 1. `yarn install` - installs dependencies including downloading duckdb 2. `yarn build:ts` - builds typescript 3. `yarn test` - runs all tests diff --git a/package.json b/package.json index 76d0e8c..46a0499 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,10 @@ "prettier:check": "prettier --check '**/*.{js,json,md,ts,yml}'", "prettier:fix": "prettier --write '**/*.{js,json,md,ts,yml}'", "postpublish": "yarn prebuild:upload", - "test": "yarn build:ts && NODE_OPTIONS='--max-old-space-size=8192' jest --runInBand --testTimeout=60000" + "test": "yarn setup-test-environment && yarn test:run", + "test:ci": "yarn test:run", + "test:run": "yarn build:ts && jest --max-workers=1", + "setup-test-environment": "./scripts/setup-test-environment.sh" }, "resolutions": { "cmake-js/yargs/y18n": ">=5.0.5||>=4.0.1 <5.0.0||>=3.2.2 <4.0.0", diff --git a/scripts/setup-test-environment.sh b/scripts/setup-test-environment.sh new file mode 100755 index 0000000..231e0e6 --- /dev/null +++ b/scripts/setup-test-environment.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +export AWS_ACCESS_KEY_ID=$(aws configure get default.aws_access_key_id) +export AWS_SECRET_ACCESS_KEY=$(aws configure get default.aws_access_key_id) +export AWS_SESSION_TOKEN=$(aws configure get default.aws_access_key_id) diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index e8f835b..e0e546f 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -3,6 +3,7 @@ import { IExecuteOptions, RowResultFormat } from "@addon-types"; const executeOptions: IExecuteOptions = { rowResultFormat: RowResultFormat.Array }; +jest.setTimeout(60 * 1000 * 2); describe("Http/s3 interface", () => { let db: DuckDB; let connection: Connection; @@ -36,13 +37,18 @@ describe("Http/s3 interface", () => { ]); }); - // Note, this test uses "dforsber-duckdb-test" user in dev - // (won't pass with your regular DC creds since they don't have permissions to the bucket - // and it also needs the session token) + /** + * - test needs AWS creds which have the permission to read from "amazon-reviews-pds" bucket + * - when running in github actions "dforsber-duckdb-test" (Dev environment) user is used + * - when running locally "setup-test-environment" script loads credentials from your aws setup + */ it("allows reading from s3 - github actions", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); + if (process.env.AWS_SESSION_TOKEN) { + await connection.executeIterator(`SET s3_session_token='${process.env.AWS_SESSION_TOKEN}'`); + } const result = await connection.executeIterator( "SELECT * FROM parquet_scan('s3://amazon-reviews-pds/parquet/product_category=Books/part-00000-495c48e6-96d6-4650-aa65-3c36a3516ddd.c000.snappy.parquet') LIMIT 1", executeOptions, @@ -65,31 +71,4 @@ describe("Http/s3 interface", () => { 2015, ]); }); - - // TODO: merge the two s3 tests in one by using the same s3 bucket but different users - // Use DC Dev creds for this test (need to export AWS_ACCESS_KEY_ID, etc) - // Note, won't pass in github actions since that user doesn't have DC Dev perms - it.skip("allows reading from s3 - local", async () => { - await connection.executeIterator(`SET s3_region='us-east-1'`); - await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); - await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`); - await connection.executeIterator(`SET s3_session_token='${process.env.AWS_SESSION_TOKEN}'`); - const result = await connection.executeIterator( - "SELECT * FROM parquet_scan('s3://node-duckdb-test-bucket/alltypes_plain.parquet') LIMIT 1", - executeOptions, - ); - expect(result.fetchRow()).toMatchObject([ - 4, - true, - 0, - 0, - 0, - 0n, - 0, - 0, - Buffer.from("03/01/09"), - Buffer.from("0"), - 1235865600000, - ]); - }); }); From 0c103a8d542b3b9247271b3a328f69e8aeaed63b Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 19:21:10 +0300 Subject: [PATCH 08/11] local test refactor --- package.json | 5 ++--- scripts/run-tests-locally.sh | 8 ++++++++ scripts/setup-test-environment.sh | 5 ----- src/tests/http-s3.test.ts | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100755 scripts/run-tests-locally.sh delete mode 100755 scripts/setup-test-environment.sh diff --git a/package.json b/package.json index 46a0499..00a6d12 100644 --- a/package.json +++ b/package.json @@ -55,10 +55,9 @@ "prettier:check": "prettier --check '**/*.{js,json,md,ts,yml}'", "prettier:fix": "prettier --write '**/*.{js,json,md,ts,yml}'", "postpublish": "yarn prebuild:upload", - "test": "yarn setup-test-environment && yarn test:run", + "test": "AWS_PROFILE=dev ./scripts/run-tests-locally.sh", "test:ci": "yarn test:run", - "test:run": "yarn build:ts && jest --max-workers=1", - "setup-test-environment": "./scripts/setup-test-environment.sh" + "test:run": "yarn build:ts && jest --max-workers=1" }, "resolutions": { "cmake-js/yargs/y18n": ">=5.0.5||>=4.0.1 <5.0.0||>=3.2.2 <4.0.0", diff --git a/scripts/run-tests-locally.sh b/scripts/run-tests-locally.sh new file mode 100755 index 0000000..d712730 --- /dev/null +++ b/scripts/run-tests-locally.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id --profile $AWS_PROFILE) +export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile $AWS_PROFILE) +export AWS_SESSION_TOKEN=$(aws configure get aws_session_token --profile $AWS_PROFILE) + +yarn test:run $1 \ No newline at end of file diff --git a/scripts/setup-test-environment.sh b/scripts/setup-test-environment.sh deleted file mode 100755 index 231e0e6..0000000 --- a/scripts/setup-test-environment.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e -export AWS_ACCESS_KEY_ID=$(aws configure get default.aws_access_key_id) -export AWS_SECRET_ACCESS_KEY=$(aws configure get default.aws_access_key_id) -export AWS_SESSION_TOKEN=$(aws configure get default.aws_access_key_id) diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index e0e546f..307f5dd 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -40,7 +40,7 @@ describe("Http/s3 interface", () => { /** * - test needs AWS creds which have the permission to read from "amazon-reviews-pds" bucket * - when running in github actions "dforsber-duckdb-test" (Dev environment) user is used - * - when running locally "setup-test-environment" script loads credentials from your aws setup + * - when running locally the "run-tests-locally" script loads credentials from your aws setup */ it("allows reading from s3 - github actions", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); From 4af007fadf9492d4b95c48a2885d9c3885d5af0c Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 19:53:27 +0300 Subject: [PATCH 09/11] empty From 8136a16930641f4d53b6d2f38663cdb68ff86add Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 19:57:37 +0300 Subject: [PATCH 10/11] dockerfile --- Dockerfile | 1 + docker-compose.yml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98deb44..2946da4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM amazon/aws-lambda-nodejs:14 WORKDIR /app RUN yum update -y && \ + yum -y install aws-cli && \ yum -y install openssl11-devel && \ yum install make -y && \ yum install yum install gcc-c++ -y && \ diff --git a/docker-compose.yml b/docker-compose.yml index a23e302..e2b1955 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,12 @@ services: environment: YARN_CACHE_FOLDER: /yarn_cache GITHUB_TOKEN: $GITHUB_TOKEN - AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN: $AWS_SESSION_TOKEN volumes: - .:/root/node-duckdb - ~/.npmrc:/root/.npmrc - node-modules:/node_modules - yarn-cache:/yarn_cache + - ~/.aws/credentials:/root/.aws/credentials:ro entrypoint: ["sh", "-c"] command: ["yarn install && yarn prebuild:current-target"] From 3eb7fe47307ba5f1f86a0874e902c16fd43c45a0 Mon Sep 17 00:00:00 2001 From: Rostislav Provodenko Date: Wed, 15 Dec 2021 20:04:06 +0300 Subject: [PATCH 11/11] update --- package.json | 2 +- src/tests/http-s3.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 00a6d12..e11e352 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-duckdb", - "version": "0.0.78", + "version": "0.0.79", "private": false, "description": "DuckDB for Node.JS", "keywords": [ diff --git a/src/tests/http-s3.test.ts b/src/tests/http-s3.test.ts index 307f5dd..48e44c2 100644 --- a/src/tests/http-s3.test.ts +++ b/src/tests/http-s3.test.ts @@ -42,7 +42,7 @@ describe("Http/s3 interface", () => { * - when running in github actions "dforsber-duckdb-test" (Dev environment) user is used * - when running locally the "run-tests-locally" script loads credentials from your aws setup */ - it("allows reading from s3 - github actions", async () => { + it("allows reading from s3", async () => { await connection.executeIterator(`SET s3_region='us-east-1'`); await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`); await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`);