feat: [ODIN-1435] s3 reading (#131)

.github/workflows/build.yml

@@ -35,8 +35,7 @@ jobs:
         run: |
           yarn install --frozen-lockfile
           yarn lint:check
-          yarn audit
-          yarn test
+          yarn test:ci
         env:
           NODE_AUTH_TOKEN: ${{secrets.GIT_REGISTRY_TOKEN}}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

Dockerfile

@@ -1,3 +1,14 @@
-FROM 059037012730.dkr.ecr.us-east-1.amazonaws.com/deepcrawl/odin-api:latest
+FROM amazon/aws-lambda-nodejs:14
 WORKDIR /app
-RUN yum update -y && yum -y install openssl11-devel
+RUN yum update -y && \
+  yum -y install aws-cli && \
+  yum -y install openssl11-devel && \
+  yum install make -y && \
+  yum install yum install gcc-c++ -y && \
+  yum install cmake3 -y && \
+  yum remove cmake -y && \
+  ln -s /usr/bin/cmake3 /usr/bin/cmake && \
+  npm install -g yarn
+RUN curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]

addon/duckdb.cc

@@ -71,7 +71,7 @@ DuckDB::DuckDB(const Napi::CallbackInfo &info)
     throw Napi::Error::New(env, e.what());
   } catch (...) {
     throw Napi::Error::New(env,
-                           "An error occured during DuckDB initialisation");
+                           "An error occurred during DuckDB initialisation");
   }
 }
 

docker-compose.yml

@@ -16,6 +16,7 @@ services:
       - ~/.npmrc:/root/.npmrc
       - node-modules:/node_modules
       - yarn-cache:/yarn_cache
+      - ~/.aws/credentials:/root/.aws/credentials:ro
     entrypoint: ["sh", "-c"]
     command: ["yarn install && yarn prebuild:current-target"]
 

docs/DEVELOPING.md

@@ -1,11 +1,24 @@
 ## Developing
 
-First build:
+### First build:
+
+Prerequisites:
+
+- aws cli tools (for the s3 test)
 
 1. `yarn install` - installs dependencies including downloading duckdb
 2. `yarn build:ts` - builds typescript
 3. `yarn test` - runs all tests
 
+### Build using DuckDB sources:
+
+Prerequisites:
+
+- OpenSSL v1.1+ (on Mac you may need to specify `OPENSSL_ROOT_DIR`, e.g. ` export OPENSSL_ROOT_DIR=/usr/local/opt/openssl`)
+
+1. `yarn download-duckdb`: downloads the version of duckdb specified in the package.json command
+2. `yarn build`: will build the DuckDB sources, the C++ addon sources and the typescript sources
+
 Other useful scripts:
 
 - `yarn build` - build everything

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-duckdb",
-  "version": "0.0.78",
+  "version": "0.0.79",
   "private": false,
   "description": "DuckDB for Node.JS",
   "keywords": [
@@ -40,12 +40,11 @@
     "clang:check": "yarn clang-format --dry-run --Werror addon/**",
     "clang:fix": "yarn clang-format -i addon/**",
     "cleanup:binaries": "rm -rf build prebuilds duckdb",
-    "download-duckdb": "rm -rf duckdb && curl -L https://github.com/cwida/duckdb/archive/v0.2.8.tar.gz > duckdb.tar.gz && tar xf duckdb.tar.gz && mv duckdb-0.2.8 duckdb && rm duckdb.tar.gz",
-    "download-duckdb:master": "rm -rf duckdb && git clone https://github.com/duckdb/duckdb.git && cd duckdb && git checkout 31308be8cb5b8382920b6e39ae1d2923da682edc",
+    "download-duckdb": "rm -rf duckdb && curl -L https://github.com/cwida/duckdb/archive/c07e229072595e086dc0adfac72638e8cc72de40.tar.gz > duckdb.tar.gz && tar xf duckdb.tar.gz && mv duckdb-c07e229072595e086dc0adfac72638e8cc72de40 duckdb && rm duckdb.tar.gz",
     "eslint:check": "eslint --ext .js,.json,.ts ./",
     "eslint:fix": "eslint --fix --ext .js,.json,.ts ./",
     "generate-doc": "yarn build:ts && rm -rf temp etc && mkdir etc && yarn api-extractor run --local --verbose &&  yarn api-documenter markdown -i temp -o docs/api && ./docs/replace.sh",
-    "install": "prebuild-install --verbose -d -r napi || (yarn download-duckdb:master && yarn build:duckdb && yarn prebuild:current-target)",
+    "install": "prebuild-install --verbose -d -r napi || (yarn download-duckdb && yarn build:duckdb && yarn prebuild:current-target)",
     "lint:check": "yarn prettier:check && yarn eslint:check && yarn clang:check",
     "lint:fix": "yarn prettier:fix && yarn eslint:fix && yarn clang:fix",
     "prebuild:all-targets": "yarn install && yarn prebuild:linux",
@@ -56,7 +55,9 @@
     "prettier:check": "prettier --check '**/*.{js,json,md,ts,yml}'",
     "prettier:fix": "prettier --write '**/*.{js,json,md,ts,yml}'",
     "postpublish": "yarn prebuild:upload",
-    "test": "yarn build:ts && NODE_OPTIONS='--max-old-space-size=8192' jest --runInBand --testTimeout=60000"
+    "test": "AWS_PROFILE=dev ./scripts/run-tests-locally.sh",
+    "test:ci": "yarn test:run",
+    "test:run": "yarn build:ts && jest --max-workers=1"
   },
   "resolutions": {
     "cmake-js/yargs/y18n": ">=5.0.5||>=4.0.1 <5.0.0||>=3.2.2 <4.0.0",

scripts/run-tests-locally.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id --profile $AWS_PROFILE)
+export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile $AWS_PROFILE)
+export AWS_SESSION_TOKEN=$(aws configure get aws_session_token --profile $AWS_PROFILE)
+
+yarn test:run $1

src/tests/execute-error-handling.test.ts

@@ -1,9 +1,5 @@
 import { Connection, DuckDB } from "@addon";
 
-const invalidQueryError = `Parser Error: syntax error at or near "an"
-LINE 1: an invalid query
-        ^`;
-
 describe("executeIterator method error handling", () => {
   let db: DuckDB;
   let connection: Connection;
@@ -23,9 +19,14 @@ describe("executeIterator method error handling", () => {
     });
   });
   it("correctly handles an invalid query", async () => {
-    await expect(connection.executeIterator("an invalid query")).rejects.toMatchObject({
-      message: invalidQueryError,
-    });
+    let error: Error;
+    try {
+      await connection.executeIterator("an invalid query");
+    } catch (e) {
+      error = <Error>e;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    expect(error!.message).toContain("an invalid query");
   });
   it("correctly handles a failing query - file does not exist", async () => {
     await expect(connection.executeIterator("SELECT * FROM read_csv_auto('/idontexist.csv')")).rejects.toMatchObject({

src/tests/http-s3.test.ts

@@ -3,6 +3,7 @@ import { IExecuteOptions, RowResultFormat } from "@addon-types";
 
 const executeOptions: IExecuteOptions = { rowResultFormat: RowResultFormat.Array };
 
+jest.setTimeout(60 * 1000 * 2);
 describe("Http/s3 interface", () => {
   let db: DuckDB;
   let connection: Connection;
@@ -36,14 +37,20 @@ describe("Http/s3 interface", () => {
     ]);
   });
 
+  /**
+   * - test needs AWS creds which have the permission to read from "amazon-reviews-pds" bucket
+   * - when running in github actions "dforsber-duckdb-test" (Dev environment) user is used
+   * - when running locally the "run-tests-locally" script loads credentials from your aws setup
+   */
   it("allows reading from s3", async () => {
     await connection.executeIterator(`SET s3_region='us-east-1'`);
     await connection.executeIterator(`SET s3_access_key_id='${process.env.AWS_ACCESS_KEY_ID}'`);
     await connection.executeIterator(`SET s3_secret_access_key='${process.env.AWS_SECRET_ACCESS_KEY}'`);
-    // NOTE: The "=" sign had to be encoded to "%3D" to make this work,
-    //       probably something that DuckDB should be doing!
+    if (process.env.AWS_SESSION_TOKEN) {
+      await connection.executeIterator(`SET s3_session_token='${process.env.AWS_SESSION_TOKEN}'`);
+    }
     const result = await connection.executeIterator(
-      "SELECT * FROM parquet_scan('s3://amazon-reviews-pds/parquet/product_category%3DBooks/part-00000-495c48e6-96d6-4650-aa65-3c36a3516ddd.c000.snappy.parquet') LIMIT 1",
+      "SELECT * FROM parquet_scan('s3://amazon-reviews-pds/parquet/product_category=Books/part-00000-495c48e6-96d6-4650-aa65-3c36a3516ddd.c000.snappy.parquet') LIMIT 1",
       executeOptions,
     );
     expect(result.fetchRow()).toMatchObject([

yarn.lock

@@ -1319,9 +1319,9 @@ ansi-regex@^4.0.0, ansi-regex@^4.1.0:
   integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
 
 ansi-regex@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
-  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
+  integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
 
 ansi-styles@^3.2.0, ansi-styles@^3.2.1:
   version "3.2.1"
@@ -1522,11 +1522,11 @@ aws4@^1.8.0:
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
 axios@^0.21.1:
-  version "0.21.1"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8"
-  integrity sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==
+  version "0.21.4"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
+  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
   dependencies:
-    follow-redirects "^1.10.0"
+    follow-redirects "^1.14.0"
 
 babel-jest@^26.6.3:
   version "26.6.3"
@@ -3518,10 +3518,10 @@ flatted@^3.1.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561"
   integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==
 
-follow-redirects@^1.10.0:
-  version "1.14.2"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.2.tgz#cecb825047c00f5e66b142f90fed4f515dec789b"
-  integrity sha512-yLR6WaE2lbF0x4K2qE2p9PEXKLDjUjnR/xmjS3wHAYxtlsI9MLLBJUZirAHKzUZDGLxje7w/cXR49WOUo4rbsA==
+follow-redirects@^1.14.0:
+  version "1.14.6"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.6.tgz#8cfb281bbc035b3c067d6cd975b0f6ade6e855cd"
+  integrity "sha1-jPsoG7wDWzwGfWzZdbD2reboVc0= sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A=="
 
 for-in@^1.0.2:
   version "1.0.2"
@@ -8137,9 +8137,9 @@ tiny-relative-date@^1.3.0:
   integrity sha512-MOQHpzllWxDCHHaDno30hhLfbouoYlOI8YlMNtvKe1zXbjEVhbcEovQxvZrPvtiYW630GQDoMMarCnjfyfHA+A==
 
 [email protected]:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
-  integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc"
+  integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==
 
 to-fast-properties@^2.0.0:
   version "2.0.0"