Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Runs bb and nargo behind sem in bootstrap #11039

Closed
wants to merge 1 commit into from
Closed

chore: Runs bb and nargo behind sem in bootstrap #11039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions noir-projects/noir-contracts/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpil
export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns)

export tmp_dir=./target/tmp
export sem_id="nargo"

# Create our tmp working directory, ensure it's removed on exit.
function on_exit() {
Expand Down Expand Up @@ -72,7 +73,7 @@ function process_function() {
if ! cache_download vk-$hash.tar.gz &> /dev/null; then
# It's not in the cache. Generate the vk file and upload it to the cache.
echo_stderr "Generating vk for function: $name..."
echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk_for_ivc -b - -o $tmp_dir/$hash 2>/dev/null
echo "$bytecode_b64" | base64 -d | gunzip | sem --jobs $PARALLELISM --fg --id $sem_id $BB write_vk_for_ivc -b - -o $tmp_dir/$hash 2>/dev/null
cache_upload vk-$hash.tar.gz $tmp_dir/$hash &> /dev/null
fi

Expand Down Expand Up @@ -105,7 +106,7 @@ function compile {
"^noir-projects/aztec-nr/" \
)"
if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then
$NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0
sem --jobs $PARALLELISM --fg --id $sem_id $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0
$TRANSPILER $json_path $json_path
cache_upload contract-$contract_hash.tar.gz $json_path &> /dev/null
fi
Expand All @@ -117,7 +118,7 @@ function compile {
# .[1] is the updated functions on stdin (-)
# * merges their fields.
jq -c '.functions[]' $json_path | \
parallel -j$PARALLELISM --keep-order -N1 --block 8M --pipe --halt now,fail=1 process_function | \
parallel -j4 --keep-order -N1 --block 8M --pipe --halt now,fail=1 process_function | \
jq -s '{functions: .}' | jq -s '.[0] * {functions: .[1].functions}' $json_path - > $tmp_dir/$filename
mv $tmp_dir/$filename $json_path
}
Expand All @@ -132,7 +133,7 @@ function build {
set +e
echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..."
grep -oP '(?<=contracts/)[^"]+' Nargo.toml | \
parallel -j$PARALLELISM --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
code=$?
cat joblog.txt
return $code
Expand Down
29 changes: 15 additions & 14 deletions noir-projects/noir-protocol-circuits/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export NARGO_HASH=$(cache_content_hash ../../noir/.rebuild_patterns)

tmp_dir=./target/tmp
key_dir=./target/keys
sem_id="nargo"

# Circuits matching these patterns we have clientivc keys computed, rather than ultrahonk.
ivc_patterns=(
Expand Down Expand Up @@ -49,7 +50,7 @@ mkdir -p $tmp_dir
mkdir -p $key_dir

# Export vars needed inside compile.
export tmp_dir key_dir ci3 ivc_regex rollup_honk_regex
export tmp_dir key_dir ci3 ivc_regex rollup_honk_regex sem_id

function compile {
set -euo pipefail
Expand All @@ -61,20 +62,20 @@ function compile {
local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2"
# echo_stderr $program_hash_cmd
program_hash=$(dump_fail "$program_hash_cmd")
echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
# echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
hash=$(hash_str "$NARGO_HASH-$program_hash")
if ! cache_download circuit-$hash.tar.gz 1>&2; then
SECONDS=0
rm -f $json_path
# TODO: --skip-brillig-constraints-check added temporarily for blobs build time.
local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check"
echo_stderr "$compile_cmd"
dump_fail "$compile_cmd"
echo_stderr "Compilation complete for: $name (${SECONDS}s)"
# echo_stderr "$compile_cmd"
echo_stderr "Compiling: $name..."
sem --jobs $PARALLELISM --fg --id $sem_id dump_fail "$compile_cmd"
echo_stderr "Compilation complete for: $name"
cache_upload circuit-$hash.tar.gz $json_path &> /dev/null
fi

echo "$name"
# echo "$name"
if echo "$name" | grep -qE "${ivc_regex}"; then
local proto="client_ivc"
local write_vk_cmd="write_vk_for_ivc"
Expand All @@ -88,7 +89,7 @@ function compile {
local write_vk_cmd="write_vk_ultra_honk -h 1"
local vk_as_fields_cmd="vk_as_fields_ultra_honk"
fi
echo "$proto$"
# echo "$proto$"

# No vks needed for simulated circuits.
[[ "$name" == *"simulated"* ]] && return
Expand All @@ -99,16 +100,16 @@ function compile {
hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto")
if ! cache_download vk-$hash.tar.gz 1>&2; then
local key_path="$key_dir/$name.vk.data.json"
echo_stderr "Generating vk for function: $name..."
SECONDS=0
echo_stderr "Generating vk for: $name..."
local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o - --recursive | xxd -p -c 0"
echo_stderr $vk_cmd
vk=$(dump_fail "$vk_cmd")
# echo_stderr $vk_cmd
vk=$(sem --jobs $PARALLELISM --fg --id $sem_id dump_fail "$vk_cmd")
local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -"
# echo_stderrr $vkf_cmd
echo_stderr "Converting vk to fields for: $name..."
vk_fields=$(dump_fail "$vkf_cmd")
jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
echo_stderr "Key output at: $key_path (${SECONDS}s)"
echo_stderr "Key output at: $key_path"
cache_upload vk-$hash.tar.gz $key_path &> /dev/null
fi
}
Expand All @@ -126,7 +127,7 @@ function build {
echo "$(basename $dir)"
fi
done | \
parallel -j$PARALLELISM --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
parallel -j32 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
code=$?
cat joblog.txt
return $code
Expand Down
Loading