feat: use s3 cache in bootstrap fast (#9111)

This PR switches the cache used by `./bootstrap.sh fast` from Dockerimages built in CircleCI to Earthly artifacts stored in an S3 bucket built during Github Action runs. The new script requires access to the `aws` command and for credentials to be set up to read from the S3 bucket. Fix #8929 --------- Co-authored-by: ludamad <[email protected]>
AztecProtocol · Oct 11, 2024 · 349f938 · 349f938
1 parent 1f53957
commit 349f938
Show file tree

Hide file tree

Showing 22 changed files with 178 additions and 94 deletions.
diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile
@@ -12,7 +12,11 @@ source:
 build:
     FROM +source
     # build avm transpiler, and make sure the big build and deps folders don't hit cache
-    RUN ./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="avm-transpiler" \
+        --rebuild_patterns="../noir/.rebuild_patterns_native .rebuild_patterns" \
+        --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \
+        --build_artifacts="target"
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
     SAVE ARTIFACT scripts/compile_then_transpile.sh
 

diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh
@@ -2,10 +2,7 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m"
-extract_repo_if_working_copy_clean avm-transpiler \
-  /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/
-
-remove_old_images avm-transpiler
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" ../build-system/s3-cache-scripts/compute-content-hash.sh)
+../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz
diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile
@@ -131,7 +131,7 @@ preset-sol:
 
 preset-wasm-threads:
   FROM +source
-  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads-v1 \
+  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads \
     --command="cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm"
   SAVE ARTIFACT build/bin
 

diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh
@@ -2,16 +2,30 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-x86_64-linux-clang \
-  /usr/src/barretenberg/cpp/build/bin ./build \
-  /usr/src/barretenberg/cpp/build-pic/lib ./build-pic
 
-echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-wasm-linux-clang \
-  /usr/src/barretenberg/cpp/build-wasm/bin ./build-wasm \
-  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./build-wasm-threads
+SCRIPTS_PATH=../../build-system/s3-cache-scripts/
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns $SCRIPTS_PATH/compute-content-hash.sh)
+TMP=$(mktemp -d)
 
-remove_old_images barretenberg-wasm-linux-clang
+function on_exit() {
+  rm -rf "$TMP"
+}
+trap on_exit EXIT
+
+# Parallel download of all the cached builds because they're quite big
+echo "
+barretenberg-preset-wasm
+barretenberg-preset-wasm-threads
+barretenberg-preset-release
+barretenberg-preset-release-world-state
+" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
+
+# # clobber the existing build with the cached build
+cp -r $TMP/barretenberg-preset-wasm/build build-wasm/
+cp -r $TMP/barretenberg-preset-wasm-threads/build build-wasm-threads/
+
+mkdir -p build
+cp -r $TMP/barretenberg-preset-release/build/* build/
+cp -r $TMP/barretenberg-preset-release-world-state/build/* build/
diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile
@@ -6,12 +6,11 @@ CACHE:
   ARG build_artifacts
   ARG prefix
   # TODO(#8929): reinstate bb.js caching
-  RUN $command
-  # DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
-  #   --prefix="bb.js-$prefix" \
-  #   --command="$command" \
-  #   --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
-  #   --build_artifacts="$build_artifacts"
+  DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="bb.js-$prefix" \
+    --command="$command" \
+    --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
+    --build_artifacts="$build_artifacts"
 
 deps:
     FROM ../../build-images+from-registry
@@ -34,7 +33,7 @@ deps:
 esm:
     FROM +deps
     DO +CACHE \
-      --prefix="browser" \
+      --prefix="esm" \
       --command="yarn build:esm" \
       --build_artifacts="."
     SAVE ARTIFACT /usr/src/barretenberg/ts build

diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh
@@ -4,9 +4,29 @@ set -eu
 cd "$(dirname "$0")"
 source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
+CACHE_SCRIPTS=../../build-system/s3-cache-scripts
+
 echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m"
-extract_repo_if_working_copy_clean bb.js /usr/src/barretenberg/ts/dest .
+TMP=$(mktemp -d)
+
+function on_exit() {
+  rm -rf "$TMP"
+}
+trap on_exit EXIT
+
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh)
+
+# Parallel download of all the cached builds because they're quite big
+echo "
+bb.js-esm
+bb.js-cjs
+bb.js-browser
+" | xargs --max-procs 0 -I {} bash -c "$CACHE_SCRIPTS/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
+
+mkdir -p dest
+cp -r $TMP/bb.js-esm/dest/* dest/
+cp -r $TMP/bb.js-cjs/dest/* dest/
+cp -r $TMP/bb.js-browser/dest/* dest/
+
 # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled.
 yarn install
-
-remove_old_images bb.js
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -128,7 +128,7 @@ elif [ "$CMD" = "full" ]; then
 elif [ "$CMD" = "fast" ]; then
   export USE_CACHE=1
   if ! can_use_ci_cache; then
-    echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}"
+    echo -e "${BOLD}${YELLOW}WARNING: AWS credentials are missing. Note this is for internal aztec devs only.${RESET}"
     exit 1
   fi
 elif [ "$CMD" = "check" ]; then

diff --git a/build-system/s3-cache-scripts/cache-download.sh b/build-system/s3-cache-scripts/cache-download.sh
@@ -1,13 +1,14 @@
 #!/bin/bash
 set -eu
 
-if [ "$#" -ne 1 ]; then
+if [ "$#" -lt 1 ]; then
     echo "Usage: $0 <tar.gz_file_to_download_and_extract>"
     exit 1
 fi
 
 # Get the tar.gz file name from the argument
 TAR_FILE="$1"
+OUT_DIR="${2:-.}"
 
 function on_exit() {
   # Cleanup the temporary tar.gz file
@@ -20,6 +21,7 @@ trap on_exit EXIT
 aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet
 
 # Extract the cache file
-tar -xzf "$TAR_FILE"
+mkdir -p "$OUT_DIR"
+tar -xzf "$TAR_FILE" -C "$OUT_DIR"
 
-echo "Cache download and extraction complete."
+echo "Cache download and extraction of $TAR_FILE complete."
diff --git a/build-system/s3-cache-scripts/cache-upload.sh b/build-system/s3-cache-scripts/cache-upload.sh
@@ -24,4 +24,4 @@ trap on_exit EXIT
 # Rest of args are our binary paths
 tar -czf "$TAR_FILE" $@
 
-aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"
+aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"
diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache
@@ -1,3 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
-type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1
+if [ ! -z ${AWS_ACCESS_KEY_ID:-} ] && [ ! -z ${AWS_SECRET_ACCESS_KEY:-} ]; then
+  exit 0
+elif [ -f ~/.aws/credentials ]; then
+  exit 0
+else
+  exit 1
+fi
diff --git a/l1-contracts/.rebuild_patterns b/l1-contracts/.rebuild_patterns
@@ -0,0 +1 @@
+^l1-contracts/src/.*\.sol$
diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile
@@ -1,15 +1,26 @@
 VERSION 0.8
 
-build:
+src:
     FROM ../build-images+from-registry
     WORKDIR /usr/src/l1-contracts
     COPY --dir lib src terraform test *.json *.toml *.sh .
-    #RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff
-    # "slither": "forge clean && forge build --build-info --skip '*/test/**' --force && slither . --checklist --ignore-compile --show-ignored-findings --config-file ./slither.config.json | tee slither_output.md",
-    # "slither-has-diff": "./slither_has_diff.sh"
+    DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns"
+
+lint:
+    FROM +src
     RUN solhint --config ./.solhint.json --fix "src/**/*.sol"
     RUN forge clean && forge fmt --check
-    RUN forge build
+
+build:
+    FROM +lint
+    WORKDIR /usr/src/l1-contracts
+
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="l1-contracts" \
+        --rebuild_patterns=".rebuild_patterns" \
+        --command="forge build" \
+        --build_artifacts="out"
+
     SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts
 
 test:

diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh
@@ -2,9 +2,7 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving contracts from remote cache...\033[0m"
-extract_repo_if_working_copy_clean l1-contracts /usr/src/l1-contracts/out .
-
-remove_old_images l1-contracts
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns ../build-system/s3-cache-scripts/compute-content-hash.sh)
+../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz
diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile
@@ -1,69 +1,82 @@
 VERSION 0.8
 
-source:
-  FROM ../build-images+from-registry
+deps:
+  LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
 
-  # Install nargo
+  FROM ../build-images+from-registry
   COPY ../noir/+nargo/nargo /usr/bin/nargo
-  # Install world state napi
-  COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+  ENV BB_HASH=$bb_source_hash
+  ENV NARGO=nargo
 
+source:
+  FROM +deps
   WORKDIR /usr/src/noir-projects
 
-  # Copy source.
-  COPY package.json yarn.lock mega_honk_circuits.json .
+  COPY package.json yarn.lock .
+  RUN yarn
+
+  COPY mega_honk_circuits.json .
   COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts .
 
 build-contracts:
   ARG RAYON_NUM_THREADS
-  LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
-  FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  FROM +source
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
 
   # Install transpiler
   COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler
+  ENV TRANSPILER=avm-transpiler
 
-  RUN yarn
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler  ./bootstrap.sh
+  WORKDIR /usr/src/noir-projects/noir-contracts
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-contracts" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT noir-contracts
 
 build-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
+  ENV PARALLEL_VK=false
+
+  WORKDIR /usr/src/noir-projects/noir-protocol-circuits
+
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json crates/autogenerated"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT noir-protocol-circuits
 
 build-mock-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN echo "building with num threads $RAYON_NUM_THREADS"
-  RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
+  ENV PARALLEL_VK=false
+
+  WORKDIR /usr/src/noir-projects/mock-protocol-circuits
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-mock-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT mock-protocol-circuits
 
 build:
@@ -109,7 +122,7 @@ format:
   FROM +source
 
   WORKDIR /usr/src/noir-projects/noir-protocol-circuits
-  RUN yarn && node ./scripts/generate_variants.js
+  RUN node ./scripts/generate_variants.js
   RUN nargo fmt --check
 
   WORKDIR /usr/src/noir-projects/mock-protocol-circuits

diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh
@@ -2,16 +2,19 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+SCRIPTS_PATH=../build-system/s3-cache-scripts/
 
 echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m"
-extract_repo_if_working_copy_clean noir-projects \
-  /usr/src/noir-projects/noir-contracts/target ./noir-contracts \
-  /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits
 
-remove_old_images noir-projects
+PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
+MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
+CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
 
-yarn
+echo "
+noir-protocol-circuits $PROTOCOL_CIRCUITS_HASH
+mock-protocol-circuits $MOCK_CIRCUITS_HASH
+noir-contracts $CONTRACTS_HASH
+" | xargs --max-procs 0 --max-args 2 bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-\$0-\$1.tar.gz \$0"
 
-./mock-protocol-circuits/bootstrap.sh
-(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
+yarn
diff --git a/noir-projects/mock-protocol-circuits/.rebuild_patterns b/noir-projects/mock-protocol-circuits/.rebuild_patterns
@@ -0,0 +1,4 @@
+^noir-projects/mock-protocol-circuits/.*\.(nr|toml)$
+^noir-projects/mock-protocol-circuits/bootstrap.sh$
+^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$
+^noir-projects/scripts/generate_vk_json.js$
diff --git a/noir-projects/noir-contracts/.rebuild_patterns b/noir-projects/noir-contracts/.rebuild_patterns
@@ -0,0 +1,4 @@
+^noir-projects/noir-contracts/.*\.(nr|toml|sh|json|js)$
+^noir-projects/aztec-nr/.*\.(nr|toml)$
+^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$
+^noir-projects/scripts/generate_vk_json.js$