From 7e462c9f7c0ee3aaa34e95d9c287f2132b7e60fa Mon Sep 17 00:00:00 2001 From: Durran Jordan Date: Fri, 29 Jul 2022 20:29:05 +0200 Subject: [PATCH] test(NODE-4463): download shared lib in evergreen (#3333) Co-authored-by: Anna Henningsen Co-authored-by: Bailey Pearson --- .evergreen/config.in.yml | 35 ++++++++++++ .evergreen/config.yml | 76 ++++++++++++++++++++++++-- .evergreen/generate_evergreen_tasks.js | 60 +++++++++++++++++++- .evergreen/prepare-crypt-shared-lib.sh | 35 ++++++++++++ .evergreen/run-custom-csfle-tests.sh | 5 ++ test/readme.md | 2 + 6 files changed, 207 insertions(+), 6 deletions(-) create mode 100755 .evergreen/prepare-crypt-shared-lib.sh diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 38d96393372..b6b42f84102 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -789,6 +789,41 @@ functions: MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + "run custom csfle shared lib tests": + - command: shell.exec + type: test + params: + silent: true + working_dir: "src" + script: | + ${PREPARE_SHELL} + cat < prepare_client_encryption.sh + export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' + export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' + export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' + export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' + export CSFLE_GIT_REF='${CSFLE_GIT_REF}' + export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}' + EOT + - command: shell.exec + type: test + params: + working_dir: "src" + timeout_secs: 60 + script: | + ${PREPARE_SHELL} + + # Disable xtrace (just in case it was accidentally set). + set +x + . ./prepare_client_encryption.sh + rm -f ./prepare_client_encryption.sh + + VERSION=${VERSION} DRIVERS_TOOLS=${DRIVERS_TOOLS} \ + bash ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh + + MONGODB_URI="${MONGODB_URI}" CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + "run custom snappy tests": - command: subprocess.exec params: diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8e5428d3cde..ac7b1b4b8d6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -754,6 +754,40 @@ functions: rm -f ./prepare_client_encryption.sh MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + run custom csfle shared lib tests: + - command: shell.exec + type: test + params: + silent: true + working_dir: src + script: | + ${PREPARE_SHELL} + cat < prepare_client_encryption.sh + export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' + export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' + export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' + export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' + export CSFLE_GIT_REF='${CSFLE_GIT_REF}' + export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}' + EOT + - command: shell.exec + type: test + params: + working_dir: src + timeout_secs: 60 + script: | + ${PREPARE_SHELL} + + # Disable xtrace (just in case it was accidentally set). + set +x + . ./prepare_client_encryption.sh + rm -f ./prepare_client_encryption.sh + + VERSION=${VERSION} DRIVERS_TOOLS=${DRIVERS_TOOLS} \ + bash ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh + + MONGODB_URI="${MONGODB_URI}" CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}" \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh run custom snappy tests: - command: subprocess.exec params: @@ -1653,7 +1687,7 @@ tasks: - func: run bson-ext test vars: NODE_LTS_NAME: erbium - - name: run-custom-csfle-tests-pinned-commit + - name: run-custom-csfle-tests-mongocryptd-pinned-commit tags: - run-custom-dependency-tests commands: @@ -1668,7 +1702,7 @@ tasks: - func: run custom csfle tests vars: CSFLE_GIT_REF: c2712248e9f4909cdad723607ea5291d2eb48b91 - - name: run-custom-csfle-tests-master + - name: run-custom-csfle-tests-mongocryptd-master tags: - run-custom-dependency-tests commands: @@ -1683,6 +1717,38 @@ tasks: - func: run custom csfle tests vars: CSFLE_GIT_REF: master + - name: run-custom-csfle-shared-lib-tests-pinned-commit + tags: + - run-custom-dependency-tests + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: erbium + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: replica_set + - func: bootstrap kms servers + - func: run custom csfle shared lib tests + vars: + VERSION: latest + CSFLE_GIT_REF: c2712248e9f4909cdad723607ea5291d2eb48b91 + - name: run-custom-csfle-shared-lib-tests-master + tags: + - run-custom-dependency-tests + commands: + - func: install dependencies + vars: + NODE_LTS_NAME: erbium + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: replica_set + - func: bootstrap kms servers + - func: run custom csfle shared lib tests + vars: + VERSION: latest + CSFLE_GIT_REF: master - name: test-latest-server-noauth tags: - latest @@ -2243,8 +2309,10 @@ buildvariants: tasks: - run-custom-snappy-tests - run-bson-ext-test - - run-custom-csfle-tests-pinned-commit - - run-custom-csfle-tests-master + - run-custom-csfle-tests-mongocryptd-pinned-commit + - run-custom-csfle-tests-mongocryptd-master + - run-custom-csfle-shared-lib-tests-pinned-commit + - run-custom-csfle-shared-lib-tests-master - name: ubuntu1804-test-serverless display_name: Serverless Test run_on: ubuntu1804-test diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 2a88dd605dd..f1c14a2cffd 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -537,7 +537,7 @@ const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({ })); oneOffFuncAsTasks.push({ - name: 'run-custom-csfle-tests-pinned-commit', + name: 'run-custom-csfle-tests-mongocryptd-pinned-commit', tags: ['run-custom-dependency-tests'], commands: [ { @@ -564,7 +564,7 @@ oneOffFuncAsTasks.push({ }); oneOffFuncAsTasks.push({ - name: 'run-custom-csfle-tests-master', + name: 'run-custom-csfle-tests-mongocryptd-master', tags: ['run-custom-dependency-tests'], commands: [ { @@ -590,6 +590,62 @@ oneOffFuncAsTasks.push({ ] }); +oneOffFuncAsTasks.push({ + name: 'run-custom-csfle-shared-lib-tests-pinned-commit', + tags: ['run-custom-dependency-tests'], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: 'latest', + TOPOLOGY: 'replica_set' + } + }, + { func: 'bootstrap kms servers' }, + { + func: 'run custom csfle shared lib tests', + vars: { + VERSION: 'latest', + CSFLE_GIT_REF: 'c2712248e9f4909cdad723607ea5291d2eb48b91' + } + } + ] +}); + +oneOffFuncAsTasks.push({ + name: 'run-custom-csfle-shared-lib-tests-master', + tags: ['run-custom-dependency-tests'], + commands: [ + { + func: 'install dependencies', + vars: { + NODE_LTS_NAME: LOWEST_LTS + } + }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: 'latest', + TOPOLOGY: 'replica_set' + } + }, + { func: 'bootstrap kms servers' }, + { + func: 'run custom csfle shared lib tests', + vars: { + VERSION: 'latest', + CSFLE_GIT_REF: 'master' + } + } + ] +}); + // TODO NODE-3897 - generate combined coverage report const coverageTask = { name: 'download and merge coverage'.split(' ').join('-'), diff --git a/.evergreen/prepare-crypt-shared-lib.sh b/.evergreen/prepare-crypt-shared-lib.sh new file mode 100755 index 00000000000..3c07de9f33c --- /dev/null +++ b/.evergreen/prepare-crypt-shared-lib.sh @@ -0,0 +1,35 @@ +MONGODB_VERSION=${VERSION} +if [ -z "$MONGODB_VERSION" ]; then + # default to latest to match behavior of run-orchestration.sh. + MONGODB_VERSION=latest +fi + +. $DRIVERS_TOOLS/.evergreen/download-mongodb.sh +get_distro +# get_distro defines $DISTRO. +echo "distro='$DISTRO' version='$MONGODB_VERSION'". +get_mongodb_download_url_for "$DISTRO" "$MONGODB_VERSION" +# get_mongodb_download_url_for defines $MONGO_CRYPT_SHARED_DOWNLOAD_URL and $EXTRACT. +if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then + echo "There is no crypt_shared library for distro='$DISTRO' and version='$MONGODB_VERSION'". + exit 1 +else + echo "Downloading crypt_shared package from $MONGO_CRYPT_SHARED_DOWNLOAD_URL" + download_and_extract_crypt_shared "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" "$EXTRACT" + CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ + -name 'mongo_crypt_v1.so' -o \ + -name 'mongo_crypt_v1.dll' -o \ + -name 'mongo_crypt_v1.dylib')" + # Expect that we always find a crypt_shared library file and set the CRYPT_SHARED_LIB_PATH + # environment variable. If we didn't, print an error message and exit. + if [ -z "$CRYPT_SHARED_LIB_PATH" ]; then + echo 'CRYPT_SHARED_LIB_PATH is empty. Exiting.' + exit 1 + fi + # If we're on Windows, convert the "cygdrive" path to Windows-style paths. + if [ "Windows_NT" = "$OS" ]; then + CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) + fi + echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" + export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} +fi diff --git a/.evergreen/run-custom-csfle-tests.sh b/.evergreen/run-custom-csfle-tests.sh index 59bed7e6ff8..e4130bf742e 100644 --- a/.evergreen/run-custom-csfle-tests.sh +++ b/.evergreen/run-custom-csfle-tests.sh @@ -65,6 +65,11 @@ export MONGODB_URI=${MONGODB_URI} export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" export TEST_CSFLE=true + +if [ -n "$CRYPT_SHARED_LIB_PATH" ]; then + export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} +fi + set +o errexit # We want to run both test suites even if the first fails npm run check:csfle DRIVER_CSFLE_TEST_RESULT=$? diff --git a/test/readme.md b/test/readme.md index 8a2df49fea6..11b69fbb258 100644 --- a/test/readme.md +++ b/test/readme.md @@ -416,6 +416,8 @@ The following steps will walk you through how to run the tests for CSFLE. The output of the tests will include sections like "Client Side Encryption Corpus," "Client Side Encryption Functional," "Client Side Encryption Prose Tests," and "Client Side Encryption." + To run the functional tests using the crypt shared library instead of mongocryptd, download the appropriate version of the crypt shared library for the enterprise server version [here](https://www.mongodb.com/download-center/enterprise/releases) and then set the location of it in the environment variable `CRYPT_SHARED_LIB_PATH`. + #### KMIP FLE support tests 1. Install virtualenv: `pip install virtualenv`