Merge remote-tracking branch 'origin/master' into lx/automate-verify-…

…honk-proof-input-gen

.noir-sync-commit

@@ -1 +1 @@
-090501dfaf7c569b1aa944856bf68ad663572ae4
+1958a7932642e2fa556a903a3186b142a70e3e48

barretenberg/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 25753f794ff4cc8e3afd8b4d7a2b3df9fe0611c1
-	parent = 4cb5c83174f55f046d7d37e4e6a4667556ac5907
+	commit = 9662b437509c62bfc367abc57f8958e7dfbdbc7c
+	parent = f0f9c73dc3212fa8d986b7c827b6b4f0cbe85494
 	method = merge
 	cmdver = 0.4.6

barretenberg/acir_tests/flows/honk_sol.sh

@@ -1,14 +1,18 @@
 #!/bin/sh
-set -eu
+set -eux
+
+VFLAG=${VERBOSE:+-v}
+BFLAG="-b ./target/program.json"
+FLAGS="-c $CRS_PATH $VFLAG"
 
 export PROOF="$(pwd)/proof"
 export PROOF_AS_FIELDS="$(pwd)/proof_fields.json"
 
 # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs
-$BIN prove_keccak_ultra_honk -o proof
-$BIN write_vk_ultra_honk -o vk
+$BIN prove_keccak_ultra_honk -o proof $FLAGS $BFLAG
+$BIN write_vk_ultra_keccak_honk -o vk $FLAGS $BFLAG
 $BIN proof_as_fields_honk -k vk -c $CRS_PATH -p $PROOF
-$BIN contract_ultra_honk -k vk -c $CRS_PATH -b ./target/program.json -o Verifier.sol
+$BIN contract_ultra_honk -k vk -c $CRS_PATH -o Verifier.sol
 
 # Export the paths to the environment variables for the js test runner
 export VERIFIER_PATH="$(pwd)/Verifier.sol"

barretenberg/acir_tests/regenerate_verify_honk_proof_inputs.sh

@@ -18,7 +18,7 @@ fi
 
 export BRANCH
 
-./reset_acir_tests.sh assert_statement_recursive
+./reset_acir_tests.sh --rebuild-nargo --programs assert_statement_recursive
 
 cd acir_tests/assert_statement_recursive
 
@@ -45,4 +45,4 @@ $BIN proof_as_fields_honk $VFLAG -c $CRS_PATH -p "./proofs/honk_$PROOF_NAME" -o
 cd ../..
 python3 update_verify_honk_proof_inputs.py
 
-./reset_acir_tests.sh verify_honk_proof
+./reset_acir_tests.sh --programs verify_honk_proof

barretenberg/acir_tests/reset_acir_tests.sh

@@ -1,24 +1,48 @@
 #!/usr/bin/env bash
+set -e
 
 # Run from within barretenberg/acir_tests
 
-# Optional argument that can be passed to rebuild.sh
-optional_arg=${1:-}
+# Initialize variables for flags
+REBUILD_NARGO_FLAG=""
+PROGRAMS=""
 
-# Clean and rebuild noir then compile the test programs
+# Parse the arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --rebuild-nargo)
+            REBUILD_NARGO_FLAG="--rebuild-nargo"
+            ;;
+        --programs)
+            shift
+            PROGRAMS="$@"
+            break  # Exit loop after collecting all programs
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+# Clean and rebuild noir, then compile the test programs if --rebuild-nargo flag is set
 cd ../../noir/noir-repo
-cargo clean
-noirup -p .
-cd test_programs
 
-# Check if the optional argument exists and pass it to rebuild.sh
-if [ -n "$optional_arg" ]; then
-    ./rebuild.sh "$optional_arg"
+if [[ -n "$REBUILD_NARGO_FLAG" ]]; then
+    cargo clean
+    noirup -p .
+fi
+
+# Rebuild test programs with rebuild.sh
+cd test_programs
+if [[ -n "$PROGRAMS" ]]; then
+    ./rebuild.sh $PROGRAMS
 else
     ./rebuild.sh
 fi
 
 # Remove and repopulate the test artifacts in bberg
 cd ../../../barretenberg/acir_tests
 rm -rf acir_tests
-./clone_test_vectors.sh
+./clone_test_vectors.sh

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -197,6 +197,7 @@ bool proveAndVerifyHonkAcirFormat(acir_format::AcirFormat constraint_system, aci
 
     // Verify Honk proof
     auto verification_key = std::make_shared<VerificationKey>(prover.instance->proving_key);
+
     Verifier verifier{ verification_key };
 
     return verifier.verify_proof(proof);
@@ -589,7 +590,6 @@ void prove_tube(const std::string& output_path)
 
     builder->add_recursive_proof(current_aggregation_object);
 
-    info("num gates in tube circuit: ", builder->get_num_gates());
     using Prover = UltraProver_<UltraFlavor>;
     using Verifier = UltraVerifier_<UltraFlavor>;
     Prover tube_prover{ *builder };
@@ -819,7 +819,7 @@ void contract(const std::string& output_path, const std::string& vk_path)
  */
 void contract_honk(const std::string& output_path, const std::string& vk_path)
 {
-    using VerificationKey = UltraFlavor::VerificationKey;
+    using VerificationKey = UltraKeccakFlavor::VerificationKey;
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
 
     auto g2_data = get_bn254_g2_data(CRS_PATH);
@@ -1457,6 +1457,9 @@ int main(int argc, char* argv[])
         } else if (command == "write_vk_ultra_honk") {
             std::string output_path = get_option(args, "-o", "./target/vk");
             write_vk_honk<UltraFlavor>(bytecode_path, output_path);
+        } else if (command == "write_vk_ultra_keccak_honk") {
+            std::string output_path = get_option(args, "-o", "./target/vk");
+            write_vk_honk<UltraKeccakFlavor>(bytecode_path, output_path);
         } else if (command == "prove_mega_honk") {
             std::string output_path = get_option(args, "-o", "./proofs/proof");
             prove_honk<MegaFlavor>(bytecode_path, witness_path, output_path);

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_circuits.hpp

@@ -23,7 +23,8 @@ template <typename Builder> void generate_basic_arithmetic_circuit(Builder& buil
     stdlib::field_t a(stdlib::witness_t(&builder, fr::random_element()));
     stdlib::field_t b(stdlib::witness_t(&builder, fr::random_element()));
     stdlib::field_t c(&builder);
-    size_t passes = (1UL << log2_num_gates) / 4 - 4;
+    // Ensure the circuit is filled but finalisation doesn't make the circuit size go to the next power of two
+    size_t passes = (1UL << log2_num_gates) / 4 - 8;
     if (static_cast<int>(passes) <= 0) {
         throw_or_abort("too few gates");
     }

barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_checker.cpp

@@ -105,15 +105,16 @@ bool UltraCircuitChecker::check_block(Builder& builder,
         if (!result) {
             return report_fail("Failed Lookup check relation at row idx = ", idx);
         }
+        result = result && check_relation<PoseidonInternal>(values, params);
+        if (!result) {
+            return report_fail("Failed PoseidonInternal relation at row idx = ", idx);
+        }
+        result = result && check_relation<PoseidonExternal>(values, params);
+        if (!result) {
+            return report_fail("Failed PoseidonExternal relation at row idx = ", idx);
+        }
+
         if constexpr (IsMegaBuilder<Builder>) {
-            result = result && check_relation<PoseidonInternal>(values, params);
-            if (!result) {
-                return report_fail("Failed PoseidonInternal relation at row idx = ", idx);
-            }
-            result = result && check_relation<PoseidonExternal>(values, params);
-            if (!result) {
-                return report_fail("Failed PoseidonExternal relation at row idx = ", idx);
-            }
             result = result && check_databus_read(values, builder);
             if (!result) {
                 return report_fail("Failed databus read at row idx = ", idx);
@@ -289,10 +290,10 @@ void UltraCircuitChecker::populate_values(
     values.q_elliptic = block.q_elliptic()[idx];
     values.q_aux = block.q_aux()[idx];
     values.q_lookup = block.q_lookup_type()[idx];
+    values.q_poseidon2_internal = block.q_poseidon2_internal()[idx];
+    values.q_poseidon2_external = block.q_poseidon2_external()[idx];
     if constexpr (IsMegaBuilder<Builder>) {
         values.q_busread = block.q_busread()[idx];
-        values.q_poseidon2_internal = block.q_poseidon2_internal()[idx];
-        values.q_poseidon2_external = block.q_poseidon2_external()[idx];
     }
 }
 

barretenberg/cpp/src/barretenberg/commitment_schemes_recursion/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(commitment_schemes_recursion commitment_schemes stdlib_primitives)
+barretenberg_module(commitment_schemes_recursion commitment_schemes stdlib_poseidon2)

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/honk_contract.hpp

@@ -270,7 +270,7 @@ struct Transcript {
     Fr beta;
     Fr gamma;
     Fr[NUMBER_OF_ALPHAS] alphas;
-    Fr[LOG_N] gateChallenges;
+    Fr[CONST_PROOF_SIZE_LOG_N] gateChallenges;
     Fr[CONST_PROOF_SIZE_LOG_N] sumCheckUChallenges;
     Fr rho;
     // Zero morph
@@ -297,7 +297,7 @@ library TranscriptLib
 
         t.gateChallenges = generateGateChallenges(t.alphas[NUMBER_OF_ALPHAS - 1]);
 
-        t.sumCheckUChallenges = generateSumcheckChallenges(proof, t.gateChallenges[LOG_N - 1]);
+        t.sumCheckUChallenges = generateSumcheckChallenges(proof, t.gateChallenges[CONST_PROOF_SIZE_LOG_N - 1]);
         t.rho = generateRhoChallenge(proof, t.sumCheckUChallenges[CONST_PROOF_SIZE_LOG_N - 1]);
 
         t.zmY = generateZMYChallenge(t.rho, proof);
@@ -385,9 +385,9 @@ library TranscriptLib
         }
     }
 
-    function generateGateChallenges(Fr previousChallenge) internal view returns(Fr[LOG_N] memory gateChallenges)
+    function generateGateChallenges(Fr previousChallenge) internal view returns(Fr[CONST_PROOF_SIZE_LOG_N] memory gateChallenges)
     {
-        for (uint256 i = 0; i < LOG_N; i++) {
+        for (uint256 i = 0; i < CONST_PROOF_SIZE_LOG_N; i++) {
             previousChallenge = FrLib.fromBytes32(keccak256(abi.encodePacked(Fr.unwrap(previousChallenge))));
             gateChallenges[i] = previousChallenge;
         }

barretenberg/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp

@@ -109,7 +109,6 @@ template <class Fq, class Fr, class T> constexpr void affine_element<Fq, Fr, T>:
         x.data[1] = Fq::modulus.data[1];
         x.data[2] = Fq::modulus.data[2];
         x.data[3] = Fq::modulus.data[3];
-
     } else {
         (*this).x = Fq::zero();
         (*this).y = Fq::zero();

barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp

@@ -701,7 +701,7 @@ TEST_F(join_split_tests, test_0_input_notes_and_detect_circuit_change)
     // The below part detects any changes in the join-split circuit
     constexpr size_t DYADIC_CIRCUIT_SIZE = 1 << 16;
 
-    constexpr uint256_t CIRCUIT_HASH("0x470358e4d91c4c5296ef788b1165b2c439cd498f49c3f99386b002753ca3d0ee");
+    constexpr uint256_t CIRCUIT_HASH("0x9170317e02f4131b84f6b4efdd3ac23e5f392d815df37750c8f05a94c64797b2");
 
     const uint256_t circuit_hash = circuit.hash_circuit();
     // circuit is finalized now

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp

@@ -57,7 +57,18 @@ typename ExecutionTrace_<Flavor>::TraceData ExecutionTrace_<Flavor>::construct_t
 
     uint32_t offset = Flavor::has_zero_row ? 1 : 0; // Offset at which to place each block in the trace polynomials
     // For each block in the trace, populate wire polys, copy cycles and selector polys
-    for (auto& block : builder.blocks.get()) {
+
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1078): remove when Keccak flavor works with Poseidon
+    // gate
+    auto get_blocks = [&]() {
+        if constexpr (!HasKeccak<Flavor>) {
+            return builder.blocks.get();
+        } else {
+            return builder.blocks.get_for_ultra_keccak();
+        }
+    };
+
+    for (auto& block : get_blocks()) {
         auto block_size = static_cast<uint32_t>(block.size());
 
         // Update wire polynomials and copy cycles
@@ -76,10 +87,12 @@ typename ExecutionTrace_<Flavor>::TraceData ExecutionTrace_<Flavor>::construct_t
 
         // Insert the selector values for this block into the selector polynomials at the correct offset
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/398): implicit arithmetization/flavor consistency
-        for (auto [selector_poly, selector] : zip_view(trace_data.selectors, block.selectors)) {
+        for (size_t selector_idx = 0; selector_idx < NUM_USED_SELECTORS; selector_idx++) {
+            auto selector_poly = trace_data.selectors[selector_idx];
+            auto selector = block.selectors[selector_idx];
             for (size_t row_idx = 0; row_idx < block_size; ++row_idx) {
                 size_t trace_row_idx = row_idx + offset;
-                selector_poly[trace_row_idx] = selector[row_idx];
+                trace_data.selectors[selector_idx][trace_row_idx] = selector[row_idx];
             }
         }
 

barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.hpp

@@ -9,16 +9,21 @@ template <class Flavor> class ExecutionTrace_ {
     using Builder = typename Flavor::CircuitBuilder;
     using Polynomial = typename Flavor::Polynomial;
     using FF = typename Flavor::FF;
-    using TrackBlocks = typename Builder::Arithmetization::TraceBlocks;
+    using TraceBlocks = typename Builder::Arithmetization::TraceBlocks;
     using Wires = std::array<SlabVector<uint32_t>, Builder::NUM_WIRES>;
     using ProvingKey = typename Flavor::ProvingKey;
 
   public:
     static constexpr size_t NUM_WIRES = Builder::NUM_WIRES;
 
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1078): Since Keccak doesn't have knowledge of Poseidon2
+    // gate yet, we ignore the two related selectors
+    static constexpr size_t NUM_USED_SELECTORS =
+        !HasKeccak<Flavor> ? Builder::Arithmetization::NUM_SELECTORS : Builder::Arithmetization::NUM_SELECTORS - 2;
+
     struct TraceData {
         std::array<Polynomial, NUM_WIRES> wires;
-        std::array<Polynomial, Builder::Arithmetization::NUM_SELECTORS> selectors;
+        std::array<Polynomial, NUM_USED_SELECTORS> selectors;
         // A vector of sets (vectors) of addresses into the wire polynomials whose values are copy constrained
         std::vector<CyclicPermutation> copy_cycles;
         uint32_t ram_rom_offset = 0;    // offset of the RAM/ROM block in the execution trace
@@ -55,7 +60,7 @@ template <class Flavor> class ExecutionTrace_ {
                 }
                 {
                     ZoneScopedN("selector initialization");
-                    for (size_t idx = 0; idx < Builder::Arithmetization::NUM_SELECTORS; ++idx) {
+                    for (size_t idx = 0; idx < NUM_USED_SELECTORS; ++idx) {
                         selectors[idx] = Polynomial(proving_key.circuit_size);
                         std::string selector_tag = builder.selector_names[idx] + "_lagrange";
                         proving_key.polynomial_store.put(selector_tag, selectors[idx].share());

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -411,6 +411,9 @@ concept IsHonkFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWit
 template <typename T> 
 concept IsUltraFlavor = IsAnyOf<T, UltraFlavor, UltraKeccakFlavor, UltraFlavorWithZK, MegaFlavor>;
 
+template <typename T>
+concept HasKeccak = IsAnyOf<T, UltraKeccakFlavor>;
+
 template <typename T>
 concept IsGoblinFlavor = IsAnyOf<T, MegaFlavor,
                                     MegaRecursiveFlavor_<UltraCircuitBuilder>,

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp

@@ -78,10 +78,4 @@ template <typename FF, size_t NUM_WIRES, size_t NUM_SELECTORS> class ExecutionTr
     void set_fixed_size(uint32_t size_in) { fixed_size = size_in; }
 };
 
-class TranslatorArith {
-  public:
-    static constexpr size_t NUM_WIRES = 81;
-    static constexpr size_t NUM_SELECTORS = 0;
-};
-
 } // namespace bb

barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp

@@ -156,25 +156,15 @@ template <typename FF_> class MegaArith {
          * conventional Ultra arithmetization
          *
          */
-        void pad_additional()
-        {
-            q_busread().emplace_back(0);
-            q_poseidon2_external().emplace_back(0);
-            q_poseidon2_internal().emplace_back(0);
-        };
+        void pad_additional() { q_busread().emplace_back(0); };
 
         /**
          * @brief Resizes all selectors which are not part of the conventional Ultra arithmetization
          * @details Facilitates reuse of Ultra gate construction functions in arithmetizations which extend the
          * conventional Ultra arithmetization
          * @param new_size
          */
-        void resize_additional(size_t new_size)
-        {
-            q_busread().resize(new_size);
-            q_poseidon2_external().resize(new_size);
-            q_poseidon2_internal().resize(new_size);
-        };
+        void resize_additional(size_t new_size) { q_busread().resize(new_size); };
     };
 
     struct TraceBlocks : public MegaTraceBlocks<MegaTraceBlock> {