From ceebf80062a84dd77dc097ce85877a2417e0128c Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 4 Nov 2021 16:22:12 +0100 Subject: [PATCH 1/4] chore: enable rosetta cache for builds Avail ourselves of the new build cache feature in cdklabs/cdk-ops#1776. Adds two new things: **A persistently cached directory** The directory `$HOME/.s3buildcache` will be stored and restored in the S3 bucket, if configured. The build can assume that files it puts in there will be availble on the next build (and on the corresponding PR build). **Cache rosetta tablet** If there is a file in the persistent cache directory for Rosetta, pass it to `jsii-rosetta` as an input. Afterwards, store whatever tablet the build produced back into the cache directory. The latter will only impact the persistent cache if done on a build that is actually configured to store the cache back, which is only the main pipeline build. --- build.sh | 9 ++++++--- buildspec-pr.yaml | 3 +++ buildspec.yaml | 4 ++++ pack.sh | 6 +----- scripts/cache-load.sh | 23 +++++++++++++++++++++++ scripts/cache-store.sh | 21 +++++++++++++++++++++ scripts/list-packages | 4 +++- scripts/run-rosetta.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 101 insertions(+), 9 deletions(-) create mode 100755 scripts/cache-load.sh create mode 100644 scripts/cache-store.sh create mode 100644 scripts/run-rosetta.sh diff --git a/build.sh b/build.sh index aae39e94ea730..88c57de627143 100755 --- a/build.sh +++ b/build.sh @@ -81,14 +81,17 @@ trap "rm -rf $MERKLE_BUILD_CACHE" EXIT if [ "$run_tests" == "true" ]; then runtarget="$runtarget+test" fi -if [ "$extract_snippets" == "true" ]; then - runtarget="$runtarget+extract" -fi echo "=============================================================================================" echo "building..." time lerna run $bail --stream $runtarget || fail +if $extract_snippets; then + # After compilation, run Rosetta (using the cache if available). + # This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'. + /bin/bash scripts/run-rosetta.sh +fi + if [ "$check_compat" == "true" ]; then /bin/bash scripts/check-api-compatibility.sh fi diff --git a/buildspec-pr.yaml b/buildspec-pr.yaml index 647b78849b3e8..13c9b635a2c4f 100644 --- a/buildspec-pr.yaml +++ b/buildspec-pr.yaml @@ -16,6 +16,9 @@ phases: # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause # memory errors. Increasing this value allows our build to more consistently succeed - (command -v sysctl || yum install -y procps-ng) && /sbin/sysctl -w vm.max_map_count=2251954 + pre_build: + commands: + - /bin/bash ./scripts/cache-load.sh build: commands: - /bin/bash ./build.sh --extract diff --git a/buildspec.yaml b/buildspec.yaml index 0fc990a17c805..94a75e2357f78 100644 --- a/buildspec.yaml +++ b/buildspec.yaml @@ -16,6 +16,9 @@ phases: # Packing the mono-libraries (monocdk & aws-cdk-lib) can cause # memory errors. Increasing this value allows our build to more consistently succeed - /sbin/sysctl -w vm.max_map_count=2251954 + pre_build: + commands: + - /bin/bash ./scripts/cache-load.sh build: commands: - 'if ${BUMP_CANDIDATE:-false}; then /bin/bash ./scripts/bump-candidate.sh; fi' @@ -25,6 +28,7 @@ phases: post_build: commands: - "[ -f .BUILD_COMPLETED ] && /bin/bash ./pack.sh" + - /bin/bash ./scripts/cache-store.sh artifacts: files: - "**/*" diff --git a/pack.sh b/pack.sh index 168a17f972406..e6386233dbf90 100755 --- a/pack.sh +++ b/pack.sh @@ -39,11 +39,7 @@ function lerna_scopes() { # Compile examples with respect to "decdk" directory, as all packages will # be symlinked there so they can all be included. echo "Extracting code samples" >&2 -node --experimental-worker $(which $ROSETTA) \ - --compile \ - --output samples.tabl.json \ - --directory packages/decdk \ - $(cat $TMPDIR/jsii.txt) +scripts/run-rosetta $TMPDIR/jsii.txt # Jsii packaging (all at once using jsii-pacmak) echo "Packaging jsii modules" >&2 diff --git a/scripts/cache-load.sh b/scripts/cache-load.sh new file mode 100755 index 0000000000000..23aff254f4f1f --- /dev/null +++ b/scripts/cache-load.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# If the environment variable S3_BUILD_CACHE is set, download and extract the +# tarball it points to into the directory $HOME/.s3buildcache. +# +set -eu + +cachedir=$HOME/.s3buildcache +mkdir -p $cachedir + +if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then + exit 0 +fi + +echo "🧳 Build cache enabled: ${S3_BUILD_CACHE}" +if ! aws s3 ls ${S3_BUILD_CACHE} > /dev/null; then + echo "🧳⚠️ Cache not found." + exit 0 +fi + +if ! (cd $cachedir && aws s3 cp ${S3_BUILD_CACHE} - | tar xzv); then + echo "🧳⚠️ Something went wrong fetching the cache. Continuing anyway." +fi diff --git a/scripts/cache-store.sh b/scripts/cache-store.sh new file mode 100644 index 0000000000000..aaf99cbca4f0f --- /dev/null +++ b/scripts/cache-store.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# If the environment variable S3_BUILD_CACHE is set, compress and upload the +# contents of $HOME/.s3buildcache to it. +# +set -eu + +cachedir=$HOME/.s3buildcache +mkdir -p $cachedir + +if [[ "${S3_BUILD_CACHE:-}" = "" ]]; then + exit 0 +fi + +echo "🧳 Storing build cache at: ${S3_BUILD_CACHE}" + +if ! (cd $cachedir && tar czv . | aws s3 cp - ${S3_BUILD_CACHE}); then + echo "🧳⚠️ Something went wrong storing the cache." +fi + +echo "🧳 Finished." diff --git a/scripts/list-packages b/scripts/list-packages index 95a2c5ecc0379..a8be3e0f64f7e 100755 --- a/scripts/list-packages +++ b/scripts/list-packages @@ -12,7 +12,9 @@ if (process.argv.length < 4) { process.exit(1); } -child_process.exec('lerna ls --toposort --json', { shell: true }, (error, stdout) => { +const lerna = path.resolve(__dirname, '..', '..', 'node_modules', '.bin', 'lerna'); + +child_process.exec(`${lerna} ls --toposort --json`, { shell: true }, (error, stdout) => { if (error) { console.error('Error: ', error); process.exit(-1); diff --git a/scripts/run-rosetta.sh b/scripts/run-rosetta.sh new file mode 100644 index 0000000000000..0148eeccc3630 --- /dev/null +++ b/scripts/run-rosetta.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Run jsii-rosetta on all jsii packages, using the S3 build cache if available. +# +# Usage: run-rosetta [PKGSFILE] +# +# If you already have a file with a list of all the JSII package directories +# in it, pass it as the first argument. Otherwise, this script will run +# 'list-packages' to determine a list itself. +set -eu +scriptdir=$(cd $(dirname $0) && pwd) + +ROSETTA=${ROSETTA:-npx jsii-rosetta} + +if [[ "${1:-}" = "" ]]; then + echo "Collecting package list..." >&2 + TMPDIR=${TMPDIR:-$(dirname $(mktemp -u))} + node $scriptdir/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt + jsii_pkgs_file=$TMPDIR/jsii.txt +else + jsii_pkgs_file=$1 +fi + +rosetta_cache_file=$HOME/.s3buildcache/rosetta-cache.tabl.json +rosetta_cache_opts="" +if [[ -f $rosetta_cache_file ]]; then + rosetta_cache_opts="--cache-from ${rosetta_cache_file}" +fi + +$ROSETTA \ + --compile \ + --output samples.tabl.json \ + $rosetta_cache_opts \ + --directory packages/decdk \ + $(cat $jsii_pkgs_file) + +if [[ -d $(dirname $rosetta_cache_file) ]]; then + # If the cache directory is available, copy the current tablet into it + cp samples.tabl.json $rosetta_cache_file +fi From 90532068a3e6879c5ef571095543d520afdfade1 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 4 Nov 2021 16:29:11 +0100 Subject: [PATCH 2/4] Add extension --- pack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack.sh b/pack.sh index e6386233dbf90..81eecafabe187 100755 --- a/pack.sh +++ b/pack.sh @@ -39,7 +39,7 @@ function lerna_scopes() { # Compile examples with respect to "decdk" directory, as all packages will # be symlinked there so they can all be included. echo "Extracting code samples" >&2 -scripts/run-rosetta $TMPDIR/jsii.txt +scripts/run-rosetta.sh $TMPDIR/jsii.txt # Jsii packaging (all at once using jsii-pacmak) echo "Packaging jsii modules" >&2 From d016f6daea58b93d75470d46d4e2c7ee29211775 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 4 Nov 2021 16:31:55 +0100 Subject: [PATCH 3/4] Do rosetta run full well after `transform` --- build.sh | 12 +----------- buildspec-pr.yaml | 7 +++++-- scripts/transform.sh | 9 +-------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/build.sh b/build.sh index 88c57de627143..64ecc043faf45 100755 --- a/build.sh +++ b/build.sh @@ -6,11 +6,10 @@ runtarget="build" run_tests="true" check_prereqs="true" check_compat="true" -extract_snippets="false" while [[ "${1:-}" != "" ]]; do case $1 in -h|--help) - echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat] [--extract]" + echo "Usage: build.sh [--no-bail] [--force|-f] [--skip-test] [--skip-prereqs] [--skip-compat]" exit 1 ;; --no-bail) @@ -28,9 +27,6 @@ while [[ "${1:-}" != "" ]]; do --skip-compat) check_compat="false" ;; - --extract) - extract_snippets="true" - ;; *) echo "Unrecognized parameter: $1" exit 1 @@ -86,12 +82,6 @@ echo "========================================================================== echo "building..." time lerna run $bail --stream $runtarget || fail -if $extract_snippets; then - # After compilation, run Rosetta (using the cache if available). - # This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'. - /bin/bash scripts/run-rosetta.sh -fi - if [ "$check_compat" == "true" ]; then /bin/bash scripts/check-api-compatibility.sh fi diff --git a/buildspec-pr.yaml b/buildspec-pr.yaml index 13c9b635a2c4f..ade1f4a9be0c6 100644 --- a/buildspec-pr.yaml +++ b/buildspec-pr.yaml @@ -21,6 +21,9 @@ phases: - /bin/bash ./scripts/cache-load.sh build: commands: - - /bin/bash ./build.sh --extract - - /bin/bash ./scripts/transform.sh --extract + - /bin/bash ./build.sh + - /bin/bash ./scripts/transform.sh + # After compilation, run Rosetta (using the cache if available). + # This will print errors, and fail the build if there are compilation errors in any packages marked as 'strict'. + - /bin/bash ./scripts/run-rosetta.sh - git diff-index --exit-code --ignore-space-at-eol --stat HEAD diff --git a/scripts/transform.sh b/scripts/transform.sh index da780c3df49ff..c0bb83b3a6edf 100755 --- a/scripts/transform.sh +++ b/scripts/transform.sh @@ -25,12 +25,11 @@ createSymlinks() { runtarget="build" run_tests="true" -extract_snippets="false" skip_build="" while [[ "${1:-}" != "" ]]; do case $1 in -h|--help) - echo "Usage: transform.sh [--skip-test/build] [--extract]" + echo "Usage: transform.sh [--skip-test/build]" exit 1 ;; --skip-test|--skip-tests) @@ -39,9 +38,6 @@ while [[ "${1:-}" != "" ]]; do --skip-build) skip_build="true" ;; - --extract) - extract_snippets="true" - ;; *) echo "Unrecognized options: $1" exit 1 @@ -52,9 +48,6 @@ done if [ "$run_tests" == "true" ]; then runtarget="$runtarget+test" fi -if [ "$extract_snippets" == "true" ]; then - runtarget="$runtarget+extract" -fi export NODE_OPTIONS="--max-old-space-size=4096 --experimental-worker ${NODE_OPTIONS:-}" From 004b7072fd3744b6f61f510fbe3a3b0eada67ab2 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 5 Nov 2021 11:00:26 +0100 Subject: [PATCH 4/4] One directory up too far --- scripts/list-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/list-packages b/scripts/list-packages index a8be3e0f64f7e..fe89f84db9410 100755 --- a/scripts/list-packages +++ b/scripts/list-packages @@ -12,7 +12,7 @@ if (process.argv.length < 4) { process.exit(1); } -const lerna = path.resolve(__dirname, '..', '..', 'node_modules', '.bin', 'lerna'); +const lerna = path.resolve(__dirname, '..', 'node_modules', '.bin', 'lerna'); child_process.exec(`${lerna} ls --toposort --json`, { shell: true }, (error, stdout) => { if (error) {