fix: memory fragmentation fixes to cut UltraHonk memory usage by 26%

barretenberg/cpp/src/barretenberg/bb/exec_pipe.hpp

@@ -8,7 +8,7 @@ inline std::vector<uint8_t> exec_pipe(std::string const& command)
 {
     FILE* pipe = popen(command.c_str(), "r");
     if (!pipe) {
-        throw std::runtime_error("popen() failed!");
+        throw std::runtime_error("popen() failed! Can't run: " + command);
     }
 
     std::vector<uint8_t> result;

barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

@@ -41,7 +41,7 @@ std::vector<g1::affine_element> get_bn254_g1_data(const std::filesystem::path& p
 
     if (g1_file_size >= num_points * 64 && g1_file_size % 64 == 0) {
         vinfo("using cached bn254 crs of size ", std::to_string(g1_file_size / 64), " at ", g1_path);
-        auto data = read_file(g1_path, g1_file_size);
+        auto data = read_file(g1_path, num_points * 64);
         auto points = std::vector<g1::affine_element>(num_points);
         for (size_t i = 0; i < num_points; ++i) {
             points[i] = from_buffer<g1::affine_element>(data, i * 64);

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -910,17 +910,23 @@ UltraProver_<Flavor> compute_valid_prover(const std::string& bytecodePath,
     } else if constexpr (IsAnyOf<Flavor, UltraRollupFlavor>) {
         honk_recursion = 2;
     }
-    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
 
-    acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion) };
-    if (!witnessPath.empty()) {
-        program.witness = get_witness(witnessPath);
-    }
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1180): Don't init grumpkin crs when unnecessary.
     init_grumpkin_crs(1 << CONST_ECCVM_LOG_N);
 
-    auto builder = acir_format::create_circuit<Builder>(program, metadata);
-    auto prover = Prover{ builder };
+    // Lambda function to ensure the builder gets freed before generating the vk. Vk generation requires initialing the
+    // pippenger runtime state which leads to it being the peak, when its functionality is purely for debugging purposes
+    // here.
+    auto prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
+        acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion) };
+        if (!witnessPath.empty()) {
+            program.witness = get_witness(witnessPath);
+        }
+        auto builder = acir_format::create_circuit<Builder>(program, metadata);
+        return Prover{ builder };
+    }();
+
     size_t required_crs_size = prover.proving_key->proving_key.circuit_size;
     if constexpr (Flavor::HasZK) {
         // Ensure there are enough points to commit to the libra polynomials required for zero-knowledge sumcheck

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -316,16 +316,18 @@ WASM_EXPORT void acir_prove_ultra_honk(uint8_t const* acir_vec,
                                        uint8_t const* witness_vec,
                                        uint8_t** out)
 {
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{
-        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
-        acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec))
-    };
+    // Lambda function to ensure things get freed before proving.
+    UltraProver prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+                                              from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
+                                          acir_format::witness_buf_to_witness_data(
+                                              from_buffer<std::vector<uint8_t>>(witness_vec)) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+
+        return UltraProver(builder);
+    }();
 
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    UltraProver prover{ builder };
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -335,14 +337,17 @@ WASM_EXPORT void acir_prove_ultra_keccak_honk(uint8_t const* acir_vec,
                                               uint8_t const* witness_vec,
                                               uint8_t** out)
 {
-    auto constraint_system =
-        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), /*honk_recursion=*/1);
-    auto witness = acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec));
-
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(
-        constraint_system, *recursive, 0, witness, /*honk_recursion=*/1);
-
-    UltraKeccakProver prover{ builder };
+    // Lambda function to ensure things get freed before proving.
+    UltraKeccakProver prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+                                              from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
+                                          acir_format::witness_buf_to_witness_data(
+                                              from_buffer<std::vector<uint8_t>>(witness_vec)) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+
+        return UltraKeccakProver(builder);
+    }();
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -381,15 +386,16 @@ WASM_EXPORT void acir_write_vk_ultra_honk(uint8_t const* acir_vec, bool const* r
 {
     using DeciderProvingKey = DeciderProvingKey_<UltraFlavor>;
     using VerificationKey = UltraFlavor::VerificationKey;
-
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
-        from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    DeciderProvingKey proving_key(builder);
+    // lambda to free the builder
+    DeciderProvingKey proving_key = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+            from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+        return DeciderProvingKey(builder);
+    }();
     VerificationKey vk(proving_key.proving_key);
+    vinfo("Constructed UltraHonk verification key");
     *out = to_heap_buffer(to_buffer(vk));
 }
 
@@ -398,14 +404,16 @@ WASM_EXPORT void acir_write_vk_ultra_keccak_honk(uint8_t const* acir_vec, bool c
     using DeciderProvingKey = DeciderProvingKey_<UltraKeccakFlavor>;
     using VerificationKey = UltraKeccakFlavor::VerificationKey;
 
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
-        from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    DeciderProvingKey proving_key(builder);
+    // lambda to free the builder
+    DeciderProvingKey proving_key = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+            from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+        return DeciderProvingKey(builder);
+    }();
     VerificationKey vk(proving_key.proving_key);
+    vinfo("Constructed UltraKeccakHonk verification key");
     *out = to_heap_buffer(to_buffer(vk));
 }
 

barretenberg/cpp/src/barretenberg/eccvm/eccvm_transcript.test.cpp

@@ -278,13 +278,16 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
 
     // Automatically generate a transcript manifest by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Check that the prover generated manifest agrees with the manifest hard coded in this suite
     auto manifest_expected = this->construct_eccvm_honk_manifest();
     auto prover_manifest = prover.transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(manifest_expected.size() > 0);
     for (size_t round = 0; round < manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_manifest[round], manifest_expected[round]) << "Prover manifest discrepency in round " << round;
     }
@@ -293,6 +296,7 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
     auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(ipa_manifest_expected.size() > 0);
     for (size_t round = 0; round < ipa_manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_ipa_manifest[round], ipa_manifest_expected[round])
             << "IPA prover manifest discrepency in round " << round;
@@ -311,6 +315,8 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
 
     // Automatically generate a transcript manifest in the prover by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Automatically generate a transcript manifest in the verifier by verifying a proof
@@ -325,10 +331,20 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
     // The last challenge generated by the ECCVM Prover is the translation univariate batching challenge and, on the
     // verifier side, is only generated in the translator verifier hence the ECCVM prover's manifest will have one extra
     // challenge
+    ASSERT(prover_manifest.size() > 0);
     for (size_t round = 0; round < prover_manifest.size() - 1; ++round) {
         ASSERT_EQ(prover_manifest[round], verifier_manifest[round])
             << "Prover/Verifier manifest discrepency in round " << round;
     }
+
+    // Check consistency of IPA transcripts
+    auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
+    auto verifier_ipa_manifest = verifier.ipa_transcript->get_manifest();
+    ASSERT(prover_ipa_manifest.size() > 0);
+    for (size_t round = 0; round < prover_ipa_manifest.size(); ++round) {
+        ASSERT_EQ(prover_ipa_manifest[round], verifier_ipa_manifest[round])
+            << "Prover/Verifier IPA manifest discrepency in round " << round;
+    }
 }
 
 /**

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.cpp

@@ -20,6 +20,9 @@ bool ECCVMVerifier::verify_proof(const ECCVMProof& proof)
     RelationParameters<FF> relation_parameters;
     transcript = std::make_shared<Transcript>(proof.pre_ipa_proof);
     ipa_transcript = std::make_shared<Transcript>(proof.ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
+
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;
 

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyVerifier_<DeciderVerificationKeys>::run_oink_verifier_on_each_in
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.cpp

@@ -34,6 +34,8 @@ ECCVMRecursiveVerifier_<Flavor>::verify_proof(const ECCVMProof& proof)
     StdlibProof<Builder> stdlib_ipa_proof = bb::convert_native_proof_to_stdlib(builder, proof.ipa_proof);
     transcript = std::make_shared<Transcript>(stdlib_proof);
     ipa_transcript = std::make_shared<Transcript>(stdlib_ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
 
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.test.cpp

@@ -102,6 +102,8 @@ template <typename RecursiveFlavor> class ECCVMRecursiveTests : public ::testing
         EXPECT_TRUE(native_result);
         auto recursive_manifest = verifier.transcript->get_manifest();
         auto native_manifest = native_verifier.transcript->get_manifest();
+
+        ASSERT(recursive_manifest.size() > 0);
         for (size_t i = 0; i < recursive_manifest.size(); ++i) {
             EXPECT_EQ(recursive_manifest[i], native_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyRecursiveVerifier_<DeciderVerificationKeys>::run_oink_verifier_o
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.test.cpp

@@ -238,6 +238,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;
@@ -316,6 +317,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/trace_to_polynomials/trace_to_polynomials.cpp

@@ -55,9 +55,11 @@ void TraceToPolynomials<Flavor>::add_memory_records_to_proving_key(TraceData& tr
     ASSERT(proving_key.memory_read_records.empty() && proving_key.memory_write_records.empty());
 
     // Update indices of RAM/ROM reads/writes based on where block containing these gates sits in the trace
+    proving_key.memory_read_records.reserve(builder.memory_read_records.size());
     for (auto& index : builder.memory_read_records) {
         proving_key.memory_read_records.emplace_back(index + trace_data.ram_rom_offset);
     }
+    proving_key.memory_write_records.reserve(builder.memory_write_records.size());
     for (auto& index : builder.memory_write_records) {
         proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset);
     }

barretenberg/cpp/src/barretenberg/transcript/transcript.hpp

@@ -150,6 +150,8 @@ template <typename TranscriptParams> class BaseTranscript {
     Fr previous_challenge{};        // default-initialized to zeros
     std::vector<Fr> current_round_data;
 
+    bool use_manifest = false; // indicates whether the manifest is turned on, currently only on for manifest tests.
+
     // "Manifest" object that records a summary of the transcript interactions
     TranscriptManifest manifest;
 
@@ -211,8 +213,10 @@ template <typename TranscriptParams> class BaseTranscript {
      */
     void consume_prover_element_frs(const std::string& label, std::span<const Fr> element_frs)
     {
-        // Add an entry to the current round of the manifest
-        manifest.add_entry(round_number, label, element_frs.size());
+        if (use_manifest) {
+            // Add an entry to the current round of the manifest
+            manifest.add_entry(round_number, label, element_frs.size());
+        }
 
         current_round_data.insert(current_round_data.end(), element_frs.begin(), element_frs.end());
 
@@ -276,6 +280,12 @@ template <typename TranscriptParams> class BaseTranscript {
         std::copy(proof.begin(), proof.end(), std::back_inserter(proof_data));
     }
 
+    /**
+     * @brief Enables the manifest
+     *
+     */
+    void enable_manifest() { use_manifest = true; }
+
     /**
      * @brief After all the prover messages have been sent, finalize the round by hashing all the data and then
      * create the number of requested challenges.
@@ -290,8 +300,10 @@ template <typename TranscriptParams> class BaseTranscript {
     {
         constexpr size_t num_challenges = sizeof...(Strings);
 
-        // Add challenge labels for current round to the manifest
-        manifest.add_challenge(round_number, labels...);
+        if (use_manifest) {
+            // Add challenge labels for current round to the manifest
+            manifest.add_challenge(round_number, labels...);
+        }
 
         // Compute the new challenge buffer from which we derive the challenges.
 
@@ -438,7 +450,13 @@ template <typename TranscriptParams> class BaseTranscript {
 
     [[nodiscard]] TranscriptManifest get_manifest() const { return manifest; };
 
-    void print() { manifest.print(); }
+    void print()
+    {
+        if (!use_manifest) {
+            info("Warning: manifest is not enabled!");
+        }
+        manifest.print();
+    }
 };
 
 template <typename Builder>

barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp

@@ -93,7 +93,7 @@ template <IsUltraFlavor Flavor> void DeciderProvingKey_<Flavor>::allocate_select
 template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_table_lookup_polynomials(const Circuit& circuit)
 {
-    PROFILE_THIS_NAME("allocate_table_lookup_polynomials_and_inverses");
+    PROFILE_THIS_NAME("allocate_table_lookup_and_lookup_read_polynomials");
 
     size_t table_offset = circuit.blocks.lookup.trace_offset;
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1193): can potentially improve memory footprint
@@ -143,6 +143,7 @@ template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_databus_polynomials(const Circuit& circuit)
     requires HasDataBus<Flavor>
 {
+    PROFILE_THIS_NAME("allocate_databus_and_lookup_inverse_polynomials");
     proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);