From 2e52314972c8445b2b213479b2576903522d849a Mon Sep 17 00:00:00 2001 From: Josh Usiskin <56369778+jusiskin@users.noreply.github.com> Date: Tue, 18 May 2021 14:20:48 -0500 Subject: [PATCH] refactor(integ): restructure integration test framework (#427) refactor(integ): restructure integration test framework - run `PRE_AWS_INTERACTION_HOOK` between each stack deploy/destroy - fix output redirection for parallel tests - unify parallel and sequential code paths - clean code organization - clean log output and organize file artifacts - update README.md with updated instructions --- integ/.eslintignore | 2 + integ/README.md | 60 +++- .../common/scripts/bash/component_e2e.sh | 14 +- .../scripts/bash/component_e2e_driver.sh | 49 +++ .../common/scripts/bash/deploy-utils.sh | 93 ++++-- integ/package.json | 3 +- integ/scripts/bash/cleanup.sh | 2 - integ/scripts/bash/deploy-infrastructure.sh | 24 +- integ/scripts/bash/report-test-results.sh | 110 +++++-- integ/scripts/bash/rfdk-integ-e2e.sh | 102 ++----- integ/scripts/bash/set-test-variables.sh | 2 - integ/scripts/bash/teardown-infrastructure.sh | 20 +- integ/scripts/node/stack-order | 2 + integ/scripts/node/stack-order.ts | 278 ++++++++++++++++++ package.json | 2 +- 15 files changed, 612 insertions(+), 151 deletions(-) create mode 100644 integ/.eslintignore create mode 100755 integ/components/deadline/common/scripts/bash/component_e2e_driver.sh create mode 100755 integ/scripts/node/stack-order create mode 100644 integ/scripts/node/stack-order.ts diff --git a/integ/.eslintignore b/integ/.eslintignore new file mode 100644 index 000000000..c3e81be67 --- /dev/null +++ b/integ/.eslintignore @@ -0,0 +1,2 @@ +node_modules +cdk.output diff --git a/integ/README.md b/integ/README.md index 3a7745548..f3de84532 100644 --- a/integ/README.md +++ b/integ/README.md @@ -3,22 +3,58 @@ To run all test suites: 1. Build and install dependencies by running build.sh from the top-level RFDK directory +1. Configure AWS credentials. There are a few options for this: + * Configure credentials [using environment variables](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html) + * Run the integration tests on an [EC2 Instance with an IAM role](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html) + * Configure credentials using the [shared credentials file](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html) +1. *[Optional]* Set the environment variable `CDK_DEFAULT_REGION` to the region the test should be deployed in (defaults to `us-west-2`) +1. Modify the `test-config.sh` configuration file. Alternatively, the same variables can be set using environment + variables, but the `SKIP_TEST_CONFIG` environment variable must be set to `true`. In bash, this can done with the + following command: -1. Configure AWS credentials (tests will use the default AWS profile, so either set up a default profile in .aws/credentials or use temporary credentials). + ```sh + export SKIP_TEST_CONFIG=true + ``` -1. Set the environment variable CDK_DEFAULT_REGION to the region the test should be deployed in + Currently the following options can be configured: + * **REQUIRED:** for the Deadline repository test component: + * `USER_ACCEPTS_SSPL_FOR_RFDK_TESTS` -1. Configure test-config.sh. This script configures which test modules will run and overrides certain default values. Currently these include: - * Options required for all Deadline test components: - * DEADLINE_VERSION - version of the Deadline repository installer used for the test - * DEADLINE_STAGING_PATH - Complete path to local staging folder for Deadline assets (see `packages/aws-rfdk/docs/DockerImageRecipes.md` for more information) - * Options required for the Deadline repository test component: - * USER_ACCEPTS_SSPL_FOR_RFDK_TESTS - should be set to true. Setting this variable is considered acceptance of the terms of the SSPL license. Follow [this link](https://www.mongodb.com/licensing/server-side-public-license) to read the terms of the SSPL license. - * Options required for the Deadline worker fleet test component (use `aws --region ec2 describe-images --owners 357466774442 --filters "Name=name,Values=*Worker*" "Name=name,Values=**" --query 'Images[*].[ImageId, Name]' --output text` to discover AMI's): - * LINUX_DEADLINE_AMI_ID - set to the ID of an available Linux worker fleet AMI with Deadline installed. - * WINDOWS_DEADLINE_AMI_ID - set to the ID of an available Windows worker fleet AMI with Deadline installed. + Should be set to `true` to accept the MongoDB SSPL. Setting this variable is + considered acceptance of the terms of the + [SSPL license](https://www.mongodb.com/licensing/server-side-public-license). + * *[Optional]* configuration for **all** Deadline test components: + * `DEADLINE_VERSION` -1. Execute `yarn run e2e` from the `integ` directory. This will handle deploying the necessary stacks, run the appropriate tests on them, and then tear them down. + Version of the Deadline repository installer used for the test + * `DEADLINE_STAGING_PATH` + + Complete path to local staging folder for Deadline assets (see + [DockerImageRecipes](../packages/aws-rfdk/docs/DockerImageRecipes.md) for more information) + * *[Optional]* configuration for the Deadline worker fleet test component: + * `LINUX_DEADLINE_AMI_ID` + + The ID of a Linux AMI that has the Deadline client installed. The Deadline version should match the version + specified in `DEADLINE_VERSION`. + * `WINDOWS_DEADLINE_AMI_ID` + + The ID of a Windows AMI that has the Deadline client installed. The Deadline version should match the version + specified in `DEADLINE_VERSION`. + +1. From the `integ` directory, run: + + yarn e2e + + This will orchestrate the integration tests including: + + 1. Deploying the CloudFormation stacks + 1. Execute tests against the stacks + 1. Tear down the CloudFormation stacks + 1. Output the results + + Testing artifacts will be persisted in the `integ/.e2etemp` directory. + Subsequent executions of the integration tests will delete this directory, + so take care to persist the artifacts if desired. # Example Output: diff --git a/integ/components/deadline/common/scripts/bash/component_e2e.sh b/integ/components/deadline/common/scripts/bash/component_e2e.sh index ac7788ed9..f84c516e2 100755 --- a/integ/components/deadline/common/scripts/bash/component_e2e.sh +++ b/integ/components/deadline/common/scripts/bash/component_e2e.sh @@ -13,16 +13,24 @@ if [[ $(basename $(pwd)) != $COMPONENT_NAME ]]; then exit 1 fi +function log_error () { + exit_code=$? + action=$1 + echo "[${COMPONENT_NAME}] ${action} failed" + return $exit_code +} + SKIP_TEST_CHECK=\$SKIP_${COMPONENT_NAME}_TEST SKIP_TEST_CHECK=$(eval "echo $SKIP_TEST_CHECK" 2> /dev/null) || SKIP_TEST_CHECK=false if [[ ! "${SKIP_TEST_CHECK}" = "true" ]]; then - # Load utility functions source "../common/scripts/bash/deploy-utils.sh" + ensure_component_artifact_dir "${COMPONENT_NAME}" + if [[ $OPTION != '--destroy-only' ]]; then - deploy_component_stacks $COMPONENT_NAME - execute_component_test $COMPONENT_NAME + deploy_component_stacks $COMPONENT_NAME || log_error "app deployment" + execute_component_test $COMPONENT_NAME || log_error "running test suite" fi if [[ $OPTION != '--deploy-and-test-only' ]]; then destroy_component_stacks $COMPONENT_NAME diff --git a/integ/components/deadline/common/scripts/bash/component_e2e_driver.sh b/integ/components/deadline/common/scripts/bash/component_e2e_driver.sh new file mode 100755 index 000000000..87c256076 --- /dev/null +++ b/integ/components/deadline/common/scripts/bash/component_e2e_driver.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Handle errors manually +set +e +# Fail on unset variables +set -u + +COMPONENT_ROOT="$1" +COMPONENT_NAME=$(basename "$COMPONENT_ROOT") +START_TIME=$SECONDS + +# Before changing directories, we determine the +# asbolute path of INTEG_TEMP_DIR, since it is a relative +# path +export INTEG_TEMP_DIR=$(readlink -fm "${INTEG_TEMP_DIR}") + +cd "$INTEG_ROOT/$COMPONENT_ROOT" + +# Ensure the component's artifact subdir exists +source "../common/scripts/bash/deploy-utils.sh" +ensure_component_artifact_dir "${COMPONENT_NAME}" + +( + set +e + ../common/scripts/bash/component_e2e.sh "$COMPONENT_NAME" + exit_code=$? + echo $exit_code > "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/exitcode" + exit $exit_code +) +test_exit_code=$? + +FINISH_TIME=$SECONDS +cat > "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/timings.sh" < "$INTEG_TEMP_DIR/${COMPONENT_NAME}_deploy.txt" 2>&1 - else - npx cdk deploy "*" --require-approval=never - fi - echo "Test app $COMPONENT_NAME deployed." + # Generate the cdk.out directory which includes a manifest.json file + # this can be used to determine the deployment ordering + echo "[${COMPONENT_NAME}] synthesizing started" + npx cdk synth &> "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/synth.log" + echo "[${COMPONENT_NAME}] synthesizing complete" + + echo "[${COMPONENT_NAME}] app deployment started" + + # Empty the deploy log file in case it was non-empty + deploy_log_path="${INTEG_TEMP_DIR}/${COMPONENT_NAME}/deploy.txt" + cp /dev/null "${deploy_log_path}" + + for stack in $(cdk_stack_deploy_order); do + run_aws_interaction_hook + + echo "[${COMPONENT_NAME}] -> [${stack}] stack deployment started" + npx cdk deploy --app cdk.out --require-approval=never -e "${stack}" &>> "${deploy_log_path}" + echo "[${COMPONENT_NAME}] -> [${stack}] stack deployment complete" + done + + echo "[${COMPONENT_NAME}] app deployment complete" return 0 } +function cdk_stack_deploy_order () { + # Outputs the stacks in topological deploy order + "${INTEG_ROOT}/scripts/node/stack-order" +} + +function cdk_stack_destroy_order () { + # Outputs the stacks in topological destroy order + "${INTEG_ROOT}/scripts/node/stack-order" -r +} + function execute_component_test () { COMPONENT_NAME=$1 run_aws_interaction_hook - echo "Running test suite $COMPONENT_NAME..." - if [ "${RUN_TESTS_IN_PARALLEL-}" = true ]; then - yarn run test "$COMPONENT_NAME.test" --json --outputFile="$INTEG_TEMP_DIR/$COMPONENT_NAME.json" > "$INTEG_TEMP_DIR/${COMPONENT_NAME}.txt" 2>&1 + test_report_path="${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json" + test_output_path="${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-output.txt" + + echo "[${COMPONENT_NAME}] running test suite started" + ensure_component_artifact_dir "${COMPONENT_NAME}" + yarn run test "$COMPONENT_NAME.test" --json --outputFile="${test_report_path}" &> "${test_output_path}" + echo "[${COMPONENT_NAME}] running test suite complete" + + + if [[ -f "${test_report_path}" && $(node -pe "require('${test_report_path}').numFailedTests") -eq 0 ]] + then + echo "[${COMPONENT_NAME}] test suite passed" else - yarn run test "$COMPONENT_NAME.test" --json --outputFile="$INTEG_TEMP_DIR/$COMPONENT_NAME.json" + echo "[${COMPONENT_NAME}] test suite failed" fi - echo "Test suite $COMPONENT_NAME complete." return 0 } @@ -49,17 +87,26 @@ function execute_component_test () { function destroy_component_stacks () { COMPONENT_NAME=$1 - run_aws_interaction_hook + ensure_component_artifact_dir "${COMPONENT_NAME}" - echo "Destroying test app $COMPONENT_NAME..." - if [ "${RUN_TESTS_IN_PARALLEL-}" = true ]; then - npx cdk destroy "*" -f > "$INTEG_TEMP_DIR/${COMPONENT_NAME}_destroy.txt" 2>&1 - else - npx cdk destroy "*" -f - fi + echo "[${COMPONENT_NAME}] app destroy started" + + destroy_log_path="${INTEG_TEMP_DIR}/${COMPONENT_NAME}/destroy.txt" + # Empty the destroy log file in case it was non-empty + cp /dev/null "${destroy_log_path}" + for stack in $(cdk_stack_destroy_order); do + run_aws_interaction_hook + + echo "[${COMPONENT_NAME}] -> [${stack}] stack destroy started" + npx cdk destroy --app cdk.out -e -f "${stack}" &>> "${destroy_log_path}" + echo "[${COMPONENT_NAME}] -> [${stack}] stack destroy complete" + done + + # Clean up artifacts rm -f "./cdk.context.json" rm -rf "./cdk.out" - echo "Test app $COMPONENT_NAME destroyed." + + echo "[${COMPONENT_NAME}] app destroy complete" return 0 } diff --git a/integ/package.json b/integ/package.json index b70a0d9f9..ab14435ec 100644 --- a/integ/package.json +++ b/integ/package.json @@ -68,7 +68,8 @@ "eslint-plugin-license-header": "^0.2.0", "jest": "^26.6.3", "pkglint": "0.32.0", - "ts-jest": "^26.5.6" + "ts-jest": "^26.5.6", + "typescript": "~4.2.4" }, "dependencies": { "@aws-cdk/aws-autoscaling": "1.104.0", diff --git a/integ/scripts/bash/cleanup.sh b/integ/scripts/bash/cleanup.sh index 84c633a59..ec22fd0fd 100755 --- a/integ/scripts/bash/cleanup.sh +++ b/integ/scripts/bash/cleanup.sh @@ -25,5 +25,3 @@ for COMPONENT in **/cdk.json; do done rm -rf "$INTEG_ROOT/node_modules" -rm -rf "$INTEG_ROOT/stage" -rm -rf "$INTEG_ROOT/.e2etemp" diff --git a/integ/scripts/bash/deploy-infrastructure.sh b/integ/scripts/bash/deploy-infrastructure.sh index 4282c40ea..281cec0aa 100755 --- a/integ/scripts/bash/deploy-infrastructure.sh +++ b/integ/scripts/bash/deploy-infrastructure.sh @@ -9,9 +9,25 @@ shopt -s globstar # Deploy the infrastructure app, a cdk app containing only a VPC to be supplied to the following tests INFRASTRUCTURE_APP="$INTEG_ROOT/components/_infrastructure" cd "$INFRASTRUCTURE_APP" -echo "Deploying RFDK-integ infrastructure..." -npx cdk deploy "*" --require-approval=never || yarn run tear-down -echo "RFDK-integ infrastructure deployed." +mkdir -p "${INTEG_TEMP_DIR}/infrastructure" +echo "[infrastructure] deployment started" + +# Handle errors manually +set +e + +# Hide the deploy log unless something goes wrong (save the scrollback buffer) +npx cdk deploy "*" --require-approval=never &> "${INTEG_TEMP_DIR}/infrastructure/deploy.txt" +deploy_exit_code=$? + +# If an exit code was returned from the deployment, output the deploy log +if [[ $deploy_exit_code -ne 0 ]] +then + echo "[infrastructure] deployment failed" + cat "${INTEG_TEMP_DIR}/infrastructure/deploy.txt" +else + echo "[infrastructure] deployment complete" +fi + cd "$INTEG_ROOT" -exit 0 +exit $deploy_exit_code diff --git a/integ/scripts/bash/report-test-results.sh b/integ/scripts/bash/report-test-results.sh index e18fbc2ee..a87ac006d 100755 --- a/integ/scripts/bash/report-test-results.sh +++ b/integ/scripts/bash/report-test-results.sh @@ -19,34 +19,91 @@ echo "Infrastructure stack cleanup runtime: $((($INFRASTRUCTURE_DESTROY_TIME / 6 report_results () { COMPONENT_NAME=$1 - if [ $(ls "$INTEG_TEMP_DIR/$COMPONENT_NAME.json" 2> /dev/null) ]; then + echo + echo "============================================================" + echo "[${COMPONENT_NAME}]: TEST REPORT" + echo "============================================================" + echo + + # Read the test run exit code from the file + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/exitcode" ]] + then + COMPONENT_EXIT_CODE=$(cat "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/exitcode") + echo "Exit code: ${COMPONENT_EXIT_CODE}" + else + echo "Exit code: (unknown)" + COMPONENT_EXIT_CODE=1 + fi + + # If the component failed, output the deploy and destroy logs for debugging + if [[ $COMPONENT_EXIT_CODE -ne 0 ]] + then + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/deploy.txt" ]] + then + echo "----------------------------------------------" + echo "[${COMPONENT_NAME}]: Deployment Log" + echo "----------------------------------------------" + + cat "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/deploy.txt" + fi + + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/destroy.txt" ]] + then + echo "----------------------------------------------" + echo "[${COMPONENT_NAME}]: Destroy log" + echo "----------------------------------------------" + + cat "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/destroy.txt" + fi + fi + + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-output.txt" ]] + then + echo "----------------------------------------------" + echo "[${COMPONENT_NAME}]: Test output" + echo "----------------------------------------------" + + cat "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-output.txt" + fi + + + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json" ]] + then # Get test numbers from jest output - TESTS_RAN=$(node -e $'const json = require(process.argv[1]); console.log(json.numTotalTests)' "$INTEG_TEMP_DIR/$COMPONENT_NAME.json") - TESTS_PASSED=$(node -e $'const json = require(process.argv[1]); console.log(json.numPassedTests)' "$INTEG_TEMP_DIR/$COMPONENT_NAME.json") - TESTS_FAILED=$(node -e $'const json = require(process.argv[1]); console.log(json.numFailedTests)' "$INTEG_TEMP_DIR/$COMPONENT_NAME.json") - - DEPLOY_START_TIME=${COMPONENT_NAME}_START_TIME - DEPLOY_FINISH_TIME=$(node -e $'const json = require(process.argv[1]); console.log(json.startTime)' "$INTEG_TEMP_DIR/$COMPONENT_NAME.json") - DEPLOY_FINISH_TIME="${DEPLOY_FINISH_TIME:0:10}" - DESTROY_START_TIME=$(node -e $'const json = require(process.argv[1]); console.log(json.testResults[0].endTime)' "$INTEG_TEMP_DIR/$COMPONENT_NAME.json") - DESTROY_START_TIME="${DESTROY_START_TIME:0:10}" - DESTROY_FINISH_TIME=${COMPONENT_NAME}_FINISH_TIME - - # Calculate seconds from when deploy began to when test began - DEPLOY_TIME=$(( $DEPLOY_FINISH_TIME - $DEPLOY_START_TIME )) - # Calculate seconds from when deploy ended to when teardown began - TEST_TIME=$(( $DESTROY_START_TIME - $DEPLOY_FINISH_TIME )) - # Calculate seconds from when test ended to when teardown finished - DESTROY_TIME=$(( $DESTROY_FINISH_TIME - $DESTROY_START_TIME )) - - echo "Results for test component $COMPONENT_NAME: " - echo " -Tests ran:" $TESTS_RAN - echo " -Tests passed:" $TESTS_PASSED - echo " -Tests failed:" $TESTS_FAILED - echo " -Deploy runtime: $((($DEPLOY_TIME / 60) % 60))m $(($DEPLOY_TIME % 60))s" - echo " -Test suite runtime: $((($TEST_TIME / 60) % 60))m $(($TEST_TIME % 60))s" - echo " -Cleanup runtime: $((($DESTROY_TIME / 60) % 60))m $(($DESTROY_TIME % 60))s" + TESTS_RAN=$(node -e $'const json = require(process.argv[1]); console.log(json.numTotalTests)' "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json") + TESTS_PASSED=$(node -e $'const json = require(process.argv[1]); console.log(json.numPassedTests)' "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json") + TESTS_FAILED=$(node -e $'const json = require(process.argv[1]); console.log(json.numFailedTests)' "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json") + + echo "Results for test component ${COMPONENT_NAME}: " + echo " -Tests ran: ${TESTS_RAN}" + echo " -Tests passed: ${TESTS_PASSED}" + echo " -Tests failed: ${TESTS_FAILED}" + + if [[ -f "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/timings.sh" ]] + then + # File contains bash variable declaration syntax for: + # ${COMPONENT_NAME}_START_TIME + # ${COMPONENT_NAME}_FINISH_TIME + source "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/timings.sh" + + DEPLOY_START_TIME=${COMPONENT_NAME}_START_TIME + DEPLOY_FINISH_TIME=$(node -e $'const json = require(process.argv[1]); console.log(json.startTime)' "${INTEG_TEMP_DIR}/${COMPONENT_NAME}/test-report.json") + DEPLOY_FINISH_TIME="${DEPLOY_FINISH_TIME:0:10}" + DESTROY_START_TIME=$(node -e $'const json = require(process.argv[1]); console.log(json.testResults[0].endTime)' "$INTEG_TEMP_DIR/${COMPONENT_NAME}/test-report.json") + DESTROY_START_TIME="${DESTROY_START_TIME:0:10}" + DESTROY_FINISH_TIME=${COMPONENT_NAME}_FINISH_TIME + + # Calculate seconds from when deploy began to when test began + DEPLOY_TIME=$(( $DEPLOY_FINISH_TIME - $DEPLOY_START_TIME )) + # Calculate seconds from when deploy ended to when teardown began + TEST_TIME=$(( $DESTROY_START_TIME - $DEPLOY_FINISH_TIME )) + # Calculate seconds from when test ended to when teardown finished + DESTROY_TIME=$(( $DESTROY_FINISH_TIME - $DESTROY_START_TIME )) + echo " -Deploy runtime: $((($DEPLOY_TIME / 60) % 60))m $(($DEPLOY_TIME % 60))s" + echo " -Test suite runtime: $((($TEST_TIME / 60) % 60))m $(($TEST_TIME % 60))s" + echo " -Cleanup runtime: $((($DESTROY_TIME / 60) % 60))m $(($DESTROY_TIME % 60))s" + fi fi } @@ -57,7 +114,6 @@ for COMPONENT in **/cdk.json; do # Use a pattern match to exclude the infrastructure app from the results if [[ "$COMPONENT_NAME" != _* ]]; then report_results $COMPONENT_NAME - fi export ${COMPONENT_NAME}_FINISH_TIME=$SECONDS done diff --git a/integ/scripts/bash/rfdk-integ-e2e.sh b/integ/scripts/bash/rfdk-integ-e2e.sh index 190008de1..ec0cb0a4a 100755 --- a/integ/scripts/bash/rfdk-integ-e2e.sh +++ b/integ/scripts/bash/rfdk-integ-e2e.sh @@ -12,6 +12,8 @@ shopt -s globstar SCRIPT_EXIT_CODE=0 +echo "RFDK end-to-end integration tests started $(date)" + # Mark test start time export TEST_START_TIME="$(date +%s)" SECONDS=$TEST_START_TIME @@ -43,7 +45,10 @@ rm -rf $INTEG_TEMP_DIR mkdir -p $INTEG_TEMP_DIR # Stage deadline from script -$BASH_SCRIPTS/stage-deadline.sh +if [ ! -d "${DEADLINE_STAGING_PATH}" ] +then + $BASH_SCRIPTS/stage-deadline.sh +fi # Extract the Deadline version to use for Deadline installations on the farm. # Tests allow not specifying or specifying a partial version string such as "10.1.12". After staging, we @@ -64,95 +69,48 @@ fi # Mark pretest finish time export PRETEST_FINISH_TIME=$SECONDS -echo "Starting RFDK-integ end-to-end tests" - # Define cleanup function for deployment failure cleanup_on_failure () { + echo "Testing failed. Performing failure cleanup..." yarn run tear-down exit 1 } +get_component_dirs () { + # Find all "cdk.json" files (indicates parent dir is a CDK app) + find . -name "cdk.json" | \ + # Filter out node_modules + grep -v node_modules | \ + # Extract the directory name + xargs -n1 dirname | \ + # Filter out apps whose driectories begin with an underscore (_) as this + # convention indicates the app is not a test + egrep -v "^_" | \ + # Sort + sort +} + # Deploy the infrastructure app, a cdk app containing only a VPC to be supplied to the following tests $BASH_SCRIPTS/deploy-infrastructure.sh || cleanup_on_failure # Mark infrastructure deploy finish time export INFRASTRUCTURE_DEPLOY_FINISH_TIME=$SECONDS -# Pull the top level directory for each cdk app in the components directory -COMPONENTS=() -for COMPONENT in **/cdk.json; do - # In case the yarn install was done inside this integ package, there are some example cdk.json files in the aws-cdk - # package we want to avoid. - if [[ $COMPONENT == *"node_modules"* ]]; then - continue - fi - - COMPONENT_ROOT="$(dirname "$COMPONENT")" - COMPONENT_NAME=$(basename "$COMPONENT_ROOT") - - # Use a pattern match to exclude the infrastructure app from the results - export ${COMPONENT_NAME}_START_TIME=$SECONDS - if [[ "$COMPONENT_NAME" != _* ]]; then - # Excecute the e2e test in the component's scripts directory - cd "$INTEG_ROOT/$COMPONENT_ROOT" - if [ "${RUN_TESTS_IN_PARALLEL-}" = true ]; then - ( (../common/scripts/bash/component_e2e.sh "$COMPONENT_NAME"; exit_code=$?; echo $exit_code > "$INTEG_TEMP_DIR/${COMPONENT_NAME}_exitcode"; exit $exit_code) \ - || ../common/scripts/bash/component_e2e.sh "$COMPONENT_NAME" --destroy-only) & - export ${COMPONENT_NAME}_PID=$! - COMPONENTS+=(${COMPONENT_NAME}) - else - ../common/scripts/bash/component_e2e.sh "$COMPONENT_NAME" || cleanup_on_failure - fi - fi - export ${COMPONENT_NAME}_FINISH_TIME=$SECONDS -done - -if [ "${RUN_TESTS_IN_PARALLEL-}" = true ]; then - while [ "${#COMPONENTS[@]}" -ne 0 ]; do - ACTIVE_COMPONENTS=() - for COMPONENT_NAME in ${COMPONENTS[@]}; do - PID=$(eval echo \"\$${COMPONENT_NAME}_PID\") - if ps -p "$PID" > /dev/null; then - ACTIVE_COMPONENTS+=(${COMPONENT_NAME}) - else - COMPONENT_EXIT_CODE=$(cat "$INTEG_TEMP_DIR/${COMPONENT_NAME}_exitcode" || echo 1) - if [ $COMPONENT_EXIT_CODE -ne 0 ]; then - SCRIPT_EXIT_CODE=1 - fi - - echo "Test app $COMPONENT_NAME finished with exit code $COMPONENT_EXIT_CODE" - if [ -f "$INTEG_TEMP_DIR/${COMPONENT_NAME}_deploy.txt" ]; then - cat "$INTEG_TEMP_DIR/${COMPONENT_NAME}_deploy.txt" - fi - if [ -f "$INTEG_TEMP_DIR/${COMPONENT_NAME}.txt" ]; then - cat "$INTEG_TEMP_DIR/${COMPONENT_NAME}.txt" - fi - if [ -f "$INTEG_TEMP_DIR/${COMPONENT_NAME}_destroy.txt" ]; then - cat "$INTEG_TEMP_DIR/${COMPONENT_NAME}_destroy.txt" - fi - fi - export ${COMPONENT_NAME}_FINISH_TIME=$SECONDS - done - if [ "${#ACTIVE_COMPONENTS[@]}" -ne 0 ]; then - COMPONENTS=(${ACTIVE_COMPONENTS[@]}) - else - COMPONENTS=() - fi - sleep 1 - done - - wait +XARGS_ARGS="-n 1" +if [[ "${RUN_TESTS_IN_PARALLEL-}" = true ]] +then + # Instruct xargs to run all the commands in parallel and block until they complete execution + XARGS_ARGS="${XARGS_ARGS} -P 0" fi -# Mark infrastructure destroy start time -export INFRASTRUCTURE_DESTROY_START_TIME=$SECONDS +# Run the component tests (potentially in parallel) +get_component_dirs | xargs ${XARGS_ARGS} components/deadline/common/scripts/bash/component_e2e_driver.sh || cleanup_on_failure # Destroy the infrastructure stack on completion cd $INTEG_ROOT +export INFRASTRUCTURE_DESTROY_START_TIME=$SECONDS # Mark infrastructure destroy start time $BASH_SCRIPTS/teardown-infrastructure.sh || cleanup_on_failure - -# Mark infrastructure destroy finish time -export INFRASTRUCTURE_DESTROY_FINISH_TIME=$SECONDS +export INFRASTRUCTURE_DESTROY_FINISH_TIME=$SECONDS # Mark infrastructure destroy finish time cd "$INTEG_ROOT" diff --git a/integ/scripts/bash/set-test-variables.sh b/integ/scripts/bash/set-test-variables.sh index 55b039a23..f2790a36e 100755 --- a/integ/scripts/bash/set-test-variables.sh +++ b/integ/scripts/bash/set-test-variables.sh @@ -3,8 +3,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -set -euo pipefail - echo "Setting test variables..." # Get region from CDK_DEFAULT_REGION; assume us-west-2 if it's not set diff --git a/integ/scripts/bash/teardown-infrastructure.sh b/integ/scripts/bash/teardown-infrastructure.sh index 1911d0a58..db269c1b0 100755 --- a/integ/scripts/bash/teardown-infrastructure.sh +++ b/integ/scripts/bash/teardown-infrastructure.sh @@ -7,13 +7,25 @@ set -euo pipefail source "$INTEG_ROOT/components/deadline/common/scripts/bash/deploy-utils.sh" -echo "Test suites completed. Destroying infrastructure stack..." +echo "[infrastructure] destroy started" INFRASTRUCTURE_APP="$INTEG_ROOT/components/_infrastructure" cd "$INFRASTRUCTURE_APP" run_aws_interaction_hook -npx cdk destroy "*" -f -echo "Infrastructure stack destroyed." +mkdir -p "${INTEG_TEMP_DIR}/infrastructure" -exit 0 +# Hide the deploy log unless something goes wrong (save the scrollback buffer) +npx cdk destroy "*" -f &> "${INTEG_TEMP_DIR}/infrastructure/destroy.txt" +destroy_exit_code=$? + +# If an exit code was returned from the deployment, output the deploy log +if [[ $destroy_exit_code -ne 0 ]] +then + echo "[infrastructure] deployment failed" + cat "${INTEG_TEMP_DIR}/infrastructure/destroy.txt" +else + echo "[infrastructure] deployment complete" +fi + +exit $destroy_exit_code diff --git a/integ/scripts/node/stack-order b/integ/scripts/node/stack-order new file mode 100755 index 000000000..6f42bba0a --- /dev/null +++ b/integ/scripts/node/stack-order @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('./stack-order.js'); diff --git a/integ/scripts/node/stack-order.ts b/integ/scripts/node/stack-order.ts new file mode 100644 index 000000000..4c8c97c27 --- /dev/null +++ b/integ/scripts/node/stack-order.ts @@ -0,0 +1,278 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * This is a command-line tool that parses a synthesized CDK manifest file and outputs the correct stack deployment or + * destroy order based on the stack dependencies. + * + * Stack names are output one per line for ease of use in shell scripts. + */ + +/* eslint-disable no-console */ + +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * The default path to look for the CDK cloud assembly manifest. + */ +const DEFAULT_MANIFEST_PATH = 'cdk.out/manifest.json'; + +/** + * Represents which type of stack ordering is desired. + */ +enum OrderType { + /** + * Output the stacks in their correct order for a CDK deployment. + * + * Stacks without dependency stacks are output first. + */ + DEPLOY, + + /** + * Output the stacks in their correct order for a CDK destroy. + * + * Stacks without dependency stacks are output last. + */ + DESTROY, +} + +/** + * Parsed program arguments as specified from the command-line. + */ +interface ProgramArguments { + /** + * The path to a cdk.out/manifest.json file of a synthesized CDK application + */ + readonly manifestPath: string; + + /** + * Whether the user desires the stack deployment order or stack destroy order + */ + readonly orderType: OrderType; +} + +/** + * A partial definition of an artifact definition within a CDK manifest file + */ +interface Artifact { + /** + * The type of artifact + */ + readonly type: string; + /** + * The key name of other artifacts that this artifact depends on + */ + readonly dependencies?: string[]; +} + +/** + * A partial JSON schema of a manifest.json file synthesized by CDK. + */ +interface Manifest { + /** + * The artifacts listed in the manifest + */ + readonly artifacts: Record; +} + +/** + * A minimal internal representation of a CDK stack + */ +interface Stack { + /** + * The stack name + */ + readonly name: string; + + /** + * The names of the stacks that this stack depends on. + */ + readonly dependencies: string[]; +} + +/** + * Returns the command-line usage of this tool suitable for console output. + */ +function usage() { + const baseName = path.basename(process.argv[1]); + return `Usage: + ${baseName} [-r] [MANIFEST_PATH] +Arguments: + MANIFEST_PATH + The path to CDK's synthesized manifest.json file. By default, CDK writes + this file to a directory named "cdk.out" in the root of the CDK app. + + If not specified this defaults to "${DEFAULT_MANIFEST_PATH}". + -r + Reverses the order. Use this to output the stack destroy order. If not + specified, the default is to output stack deploy order.`; +} + +/** + * Processes the command-line arguments and returns a parsed representation. + * + * Throws an `Error` with a user-facing error message if arguments are invalid. + */ +function parseProgramArguments(): ProgramArguments { + let orderType: OrderType = OrderType.DEPLOY; + let manifestPath: string = DEFAULT_MANIFEST_PATH; + let reverseFlag: string | undefined; + + // Strip the first two arguments (node interpreter and the path to this script) + const args = process.argv.slice(2); + + if (args.length === 2) { + // Two arguments passed. Ensure the first is the "-r" flag + [ reverseFlag, manifestPath ] = args; + + // Validate first arg is the -r flag + if (reverseFlag !== '-r') { + throw new Error(`Unexpected argument: "${reverseFlag}"`); + } + + orderType = OrderType.DESTROY; + } else if (args.length === 1) { + if (args[0] === '-r') { + orderType = OrderType.DESTROY; + } else { + // A single argument is passed containing the manifest path + manifestPath = process.argv[2]; + } + } else if (args.length > 2) { + throw new Error(`Unexpected number of arguments (${args.length})`); + } + + return { + manifestPath, + orderType, + }; +} + +/** + * An asynchronous function to read a UTF-8 encoded file. + * + * @param filePath The path of the file to be read + */ +async function readFileAsync(filePath: string) { + return new Promise((resolve, reject) => { + fs.readFile(filePath, { encoding: 'utf-8' }, (err, data) => { + if (err) { + return reject(err); + } + return resolve(data); + }); + }); +} + +/** + * Scans a parsed CDK manifest JSON structure and returns the stacks contained. + * + * @param manifest A parsed CDK manifest + */ +function findStacks(manifest: Manifest): Stack[] { + // Stacks are top-level nodes in the "artifcats" object. + return Object.entries(manifest.artifacts) + .filter(entry => entry[1].type == 'aws:cloudformation:stack') + .map(entry => { + const [ name, artifact ] = entry; + return { + name, + dependencies: artifact.dependencies ?? [], + }; + }); +} + +/** + * Orders stacks in their proper deploy/destroy order. + * + * @param stacks The stacks to be sorted + * @param orderType The type of ordering to apply + */ +function sortStacks(stacks: Stack[], orderType: OrderType): Stack[] { + /** + * A set data structure of remaining stack names to be picked. + */ + const remainingStacks: Set = new Set(stacks.map(s => s.name)); + + /** + * The sorted result array that we will accumulate stacks into + */ + let sortedStacks: Stack[] = []; + + function hasPendingDependencies(stack: Stack): boolean { + return stack.dependencies?.some(depStack => remainingStacks.has(depStack)); + } + + // Stacks with no dependencies remaining are picked on each loop iteration of the loop until there are no remaining stacks. + while(remainingStacks.size > 0) { + // Consider each remaining stack + remainingStacks.forEach(stackName => { + // Find the stack object by its name + const stack = stacks.find(val => val.name == stackName)!; + + // We can deploy this stack if it has no remaining (or un-picked) dependencies + if (!hasPendingDependencies(stack)) { + sortedStacks.push(stack); + remainingStacks.delete(stackName); + } + }); + } + + // For destroy order, we reverse the list + if (orderType == OrderType.DESTROY) { + sortedStacks = sortedStacks.reverse(); + } + + return sortedStacks; +} + +/** + * The entrypoint of the program. + * + * This processes and validates the command line arguments. It exits and + * displays an error/usage output if the arguments are invalid. + * + * If the arguments are valid, it reads the specified CDK manifest file, sorts + * the stacks in their correct deploy/destroy order, and outputs their names + * in the resulting order - one per line. + */ +async function main() { + let args: ProgramArguments; + try { + args = parseProgramArguments(); + } catch(e) { + console.error(e.toString()); + console.error(usage()); + process.exit(1); + } + const manifestRaw = await readFileAsync(args.manifestPath); + let manifest: Manifest | undefined; + + // Parse the JSON and cast to a Manifest + try { + manifest = JSON.parse(manifestRaw) as Manifest; + } catch (e) { + throw new Error(`${args.manifestPath} is not a valid JSON file`); + } + + const stacks = findStacks(manifest); + const sortedStacks = sortStacks(stacks, args.orderType); + const sortedStackNames = sortedStacks.map(stack => stack.name); + for (let stackName of sortedStackNames) { + console.log(stackName); + } +} + +main() + .catch(e => { + if (e instanceof Error) { + console.error(e.toString()); + if (e.stack) { + console.error(e.stack.toString()); + } + process.exit(1); + } + }); diff --git a/package.json b/package.json index 3999ca884..fa0bc38e2 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,7 @@ "packages/*", "packages/aws-rfdk/*", "tools/*", - "integ/" + "integ" ] } }