fix: memory fragmentation fixes to cut UltraHonk memory usage by 26% (#…

…11895)

prove_ultra_honk on the verify_honk_proof circuit goes from 3059.63MiB
to 2251.31MiB, a decrease of 26%. This gets us close to the memory
reported by tracy, 2081MiB, which doesn't account for any fragmentation
issues.

The fix hinges on a couple key issues: we want to deallocate large
objects when we don't need them anymore and we need to be careful with
our vector usage.

First, we should deallocate the builder after the prover is constructed
and before we call construct_proof, and we should also deallocate the
commitment_key during sumcheck since we do not need to commit to any
polynomials during that phase.

Second, this deallocation of the commitment key does not actually help
memory that much, in large part due to fragmentation. I discovered that
our usage of the manifest, which uses vectors for each round data,
caused tiny vectors to be littered across memory, often breaking up what
otherwise would be a large contiguous block of memory.

With this in mind, we now only use the prover manifest when we specify
that we want the manifest specifically, i.e. for the manifest tests. The
native and recursive verifier manifests will be enabled for now.

barretenberg/cpp/src/barretenberg/bb/exec_pipe.hpp

@@ -8,7 +8,7 @@ inline std::vector<uint8_t> exec_pipe(std::string const& command)
 {
     FILE* pipe = popen(command.c_str(), "r");
     if (!pipe) {
-        throw std::runtime_error("popen() failed!");
+        throw std::runtime_error("popen() failed! Can't run: " + command);
     }
 
     std::vector<uint8_t> result;

barretenberg/cpp/src/barretenberg/bb/get_bn254_crs.cpp

@@ -41,7 +41,7 @@ std::vector<g1::affine_element> get_bn254_g1_data(const std::filesystem::path& p
 
     if (g1_file_size >= num_points * 64 && g1_file_size % 64 == 0) {
         vinfo("using cached bn254 crs of size ", std::to_string(g1_file_size / 64), " at ", g1_path);
-        auto data = read_file(g1_path, g1_file_size);
+        auto data = read_file(g1_path, num_points * 64);
         auto points = std::vector<g1::affine_element>(num_points);
         for (size_t i = 0; i < num_points; ++i) {
             points[i] = from_buffer<g1::affine_element>(data, i * 64);

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -910,17 +910,23 @@ UltraProver_<Flavor> compute_valid_prover(const std::string& bytecodePath,
     } else if constexpr (IsAnyOf<Flavor, UltraRollupFlavor>) {
         honk_recursion = 2;
     }
-    const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
 
-    acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion) };
-    if (!witnessPath.empty()) {
-        program.witness = get_witness(witnessPath);
-    }
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1180): Don't init grumpkin crs when unnecessary.
     init_grumpkin_crs(1 << CONST_ECCVM_LOG_N);
 
-    auto builder = acir_format::create_circuit<Builder>(program, metadata);
-    auto prover = Prover{ builder };
+    // Lambda function to ensure the builder gets freed before generating the vk. Vk generation requires initialing the
+    // pippenger runtime state which leads to it being the peak, when its functionality is purely for debugging purposes
+    // here.
+    auto prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = recursive, .honk_recursion = honk_recursion };
+        acir_format::AcirProgram program{ get_constraint_system(bytecodePath, metadata.honk_recursion) };
+        if (!witnessPath.empty()) {
+            program.witness = get_witness(witnessPath);
+        }
+        auto builder = acir_format::create_circuit<Builder>(program, metadata);
+        return Prover{ builder };
+    }();
+
     size_t required_crs_size = prover.proving_key->proving_key.circuit_size;
     if constexpr (Flavor::HasZK) {
         // Ensure there are enough points to commit to the libra polynomials required for zero-knowledge sumcheck

barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp

@@ -316,16 +316,18 @@ WASM_EXPORT void acir_prove_ultra_honk(uint8_t const* acir_vec,
                                        uint8_t const* witness_vec,
                                        uint8_t** out)
 {
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{
-        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
-        acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec))
-    };
+    // Lambda function to ensure things get freed before proving.
+    UltraProver prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+                                              from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
+                                          acir_format::witness_buf_to_witness_data(
+                                              from_buffer<std::vector<uint8_t>>(witness_vec)) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+
+        return UltraProver(builder);
+    }();
 
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    UltraProver prover{ builder };
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -335,14 +337,17 @@ WASM_EXPORT void acir_prove_ultra_keccak_honk(uint8_t const* acir_vec,
                                               uint8_t const* witness_vec,
                                               uint8_t** out)
 {
-    auto constraint_system =
-        acir_format::circuit_buf_to_acir_format(from_buffer<std::vector<uint8_t>>(acir_vec), /*honk_recursion=*/1);
-    auto witness = acir_format::witness_buf_to_witness_data(from_buffer<std::vector<uint8_t>>(witness_vec));
-
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(
-        constraint_system, *recursive, 0, witness, /*honk_recursion=*/1);
-
-    UltraKeccakProver prover{ builder };
+    // Lambda function to ensure things get freed before proving.
+    UltraKeccakProver prover = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+                                              from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion),
+                                          acir_format::witness_buf_to_witness_data(
+                                              from_buffer<std::vector<uint8_t>>(witness_vec)) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+
+        return UltraKeccakProver(builder);
+    }();
     auto proof = prover.construct_proof();
     *out = to_heap_buffer(to_buffer</*include_size=*/true>(proof));
 }
@@ -381,15 +386,16 @@ WASM_EXPORT void acir_write_vk_ultra_honk(uint8_t const* acir_vec, bool const* r
 {
     using DeciderProvingKey = DeciderProvingKey_<UltraFlavor>;
     using VerificationKey = UltraFlavor::VerificationKey;
-
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
-        from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    DeciderProvingKey proving_key(builder);
+    // lambda to free the builder
+    DeciderProvingKey proving_key = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+            from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+        return DeciderProvingKey(builder);
+    }();
     VerificationKey vk(proving_key.proving_key);
+    vinfo("Constructed UltraHonk verification key");
     *out = to_heap_buffer(to_buffer(vk));
 }
 
@@ -398,14 +404,16 @@ WASM_EXPORT void acir_write_vk_ultra_keccak_honk(uint8_t const* acir_vec, bool c
     using DeciderProvingKey = DeciderProvingKey_<UltraKeccakFlavor>;
     using VerificationKey = UltraKeccakFlavor::VerificationKey;
 
-    const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
-
-    acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
-        from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
-    auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
-
-    DeciderProvingKey proving_key(builder);
+    // lambda to free the builder
+    DeciderProvingKey proving_key = [&] {
+        const acir_format::ProgramMetadata metadata{ .recursive = *recursive, .honk_recursion = 1 };
+        acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(
+            from_buffer<std::vector<uint8_t>>(acir_vec), metadata.honk_recursion) };
+        auto builder = acir_format::create_circuit<UltraCircuitBuilder>(program, metadata);
+        return DeciderProvingKey(builder);
+    }();
     VerificationKey vk(proving_key.proving_key);
+    vinfo("Constructed UltraKeccakHonk verification key");
     *out = to_heap_buffer(to_buffer(vk));
 }
 

barretenberg/cpp/src/barretenberg/eccvm/eccvm_transcript.test.cpp

@@ -278,13 +278,16 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
 
     // Automatically generate a transcript manifest by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Check that the prover generated manifest agrees with the manifest hard coded in this suite
     auto manifest_expected = this->construct_eccvm_honk_manifest();
     auto prover_manifest = prover.transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(manifest_expected.size() > 0);
     for (size_t round = 0; round < manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_manifest[round], manifest_expected[round]) << "Prover manifest discrepency in round " << round;
     }
@@ -293,6 +296,7 @@ TEST_F(ECCVMTranscriptTests, ProverManifestConsistency)
     auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
 
     // Note: a manifest can be printed using manifest.print()
+    ASSERT(ipa_manifest_expected.size() > 0);
     for (size_t round = 0; round < ipa_manifest_expected.size(); ++round) {
         ASSERT_EQ(prover_ipa_manifest[round], ipa_manifest_expected[round])
             << "IPA prover manifest discrepency in round " << round;
@@ -311,6 +315,8 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
 
     // Automatically generate a transcript manifest in the prover by constructing a proof
     ECCVMProver prover(builder);
+    prover.transcript->enable_manifest();
+    prover.ipa_transcript->enable_manifest();
     ECCVMProof proof = prover.construct_proof();
 
     // Automatically generate a transcript manifest in the verifier by verifying a proof
@@ -325,10 +331,20 @@ TEST_F(ECCVMTranscriptTests, VerifierManifestConsistency)
     // The last challenge generated by the ECCVM Prover is the translation univariate batching challenge and, on the
     // verifier side, is only generated in the translator verifier hence the ECCVM prover's manifest will have one extra
     // challenge
+    ASSERT(prover_manifest.size() > 0);
     for (size_t round = 0; round < prover_manifest.size() - 1; ++round) {
         ASSERT_EQ(prover_manifest[round], verifier_manifest[round])
             << "Prover/Verifier manifest discrepency in round " << round;
     }
+
+    // Check consistency of IPA transcripts
+    auto prover_ipa_manifest = prover.ipa_transcript->get_manifest();
+    auto verifier_ipa_manifest = verifier.ipa_transcript->get_manifest();
+    ASSERT(prover_ipa_manifest.size() > 0);
+    for (size_t round = 0; round < prover_ipa_manifest.size(); ++round) {
+        ASSERT_EQ(prover_ipa_manifest[round], verifier_ipa_manifest[round])
+            << "Prover/Verifier IPA manifest discrepency in round " << round;
+    }
 }
 
 /**

barretenberg/cpp/src/barretenberg/eccvm/eccvm_verifier.cpp

@@ -20,6 +20,9 @@ bool ECCVMVerifier::verify_proof(const ECCVMProof& proof)
     RelationParameters<FF> relation_parameters;
     transcript = std::make_shared<Transcript>(proof.pre_ipa_proof);
     ipa_transcript = std::make_shared<Transcript>(proof.ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
+
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;
 

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyVerifier_<DeciderVerificationKeys>::run_oink_verifier_on_each_in
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.cpp

@@ -34,6 +34,8 @@ ECCVMRecursiveVerifier_<Flavor>::verify_proof(const ECCVMProof& proof)
     StdlibProof<Builder> stdlib_ipa_proof = bb::convert_native_proof_to_stdlib(builder, proof.ipa_proof);
     transcript = std::make_shared<Transcript>(stdlib_proof);
     ipa_transcript = std::make_shared<Transcript>(stdlib_ipa_proof);
+    transcript->enable_manifest();
+    ipa_transcript->enable_manifest();
 
     VerifierCommitments commitments{ key };
     CommitmentLabels commitment_labels;

barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_verifier.test.cpp

@@ -102,6 +102,8 @@ template <typename RecursiveFlavor> class ECCVMRecursiveTests : public ::testing
         EXPECT_TRUE(native_result);
         auto recursive_manifest = verifier.transcript->get_manifest();
         auto native_manifest = native_verifier.transcript->get_manifest();
+
+        ASSERT(recursive_manifest.size() > 0);
         for (size_t i = 0; i < recursive_manifest.size(); ++i) {
             EXPECT_EQ(recursive_manifest[i], native_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.cpp

@@ -19,6 +19,7 @@ void ProtogalaxyRecursiveVerifier_<DeciderVerificationKeys>::run_oink_verifier_o
     const std::vector<FF>& proof)
 {
     transcript = std::make_shared<Transcript>(proof);
+    transcript->enable_manifest();
     size_t index = 0;
     auto key = keys_to_fold[0];
     auto domain_separator = std::to_string(index);

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.test.cpp

@@ -238,6 +238,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;
@@ -316,6 +317,7 @@ template <typename RecursiveFlavor> class ProtogalaxyRecursiveTests : public tes
         auto recursive_folding_manifest = verifier.transcript->get_manifest();
         auto native_folding_manifest = native_folding_verifier.transcript->get_manifest();
 
+        ASSERT(recursive_folding_manifest.size() > 0);
         for (size_t i = 0; i < recursive_folding_manifest.size(); ++i) {
             EXPECT_EQ(recursive_folding_manifest[i], native_folding_manifest[i])
                 << "Recursive Verifier/Verifier manifest discrepency in round " << i;

barretenberg/cpp/src/barretenberg/trace_to_polynomials/trace_to_polynomials.cpp

@@ -55,9 +55,11 @@ void TraceToPolynomials<Flavor>::add_memory_records_to_proving_key(TraceData& tr
     ASSERT(proving_key.memory_read_records.empty() && proving_key.memory_write_records.empty());
 
     // Update indices of RAM/ROM reads/writes based on where block containing these gates sits in the trace
+    proving_key.memory_read_records.reserve(builder.memory_read_records.size());
     for (auto& index : builder.memory_read_records) {
         proving_key.memory_read_records.emplace_back(index + trace_data.ram_rom_offset);
     }
+    proving_key.memory_write_records.reserve(builder.memory_write_records.size());
     for (auto& index : builder.memory_write_records) {
         proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset);
     }

barretenberg/cpp/src/barretenberg/transcript/transcript.hpp

@@ -150,6 +150,8 @@ template <typename TranscriptParams> class BaseTranscript {
     Fr previous_challenge{};        // default-initialized to zeros
     std::vector<Fr> current_round_data;
 
+    bool use_manifest = false; // indicates whether the manifest is turned on, currently only on for manifest tests.
+
     // "Manifest" object that records a summary of the transcript interactions
     TranscriptManifest manifest;
 
@@ -211,8 +213,10 @@ template <typename TranscriptParams> class BaseTranscript {
      */
     void consume_prover_element_frs(const std::string& label, std::span<const Fr> element_frs)
     {
-        // Add an entry to the current round of the manifest
-        manifest.add_entry(round_number, label, element_frs.size());
+        if (use_manifest) {
+            // Add an entry to the current round of the manifest
+            manifest.add_entry(round_number, label, element_frs.size());
+        }
 
         current_round_data.insert(current_round_data.end(), element_frs.begin(), element_frs.end());
 
@@ -276,6 +280,12 @@ template <typename TranscriptParams> class BaseTranscript {
         std::copy(proof.begin(), proof.end(), std::back_inserter(proof_data));
     }
 
+    /**
+     * @brief Enables the manifest
+     *
+     */
+    void enable_manifest() { use_manifest = true; }
+
     /**
      * @brief After all the prover messages have been sent, finalize the round by hashing all the data and then
      * create the number of requested challenges.
@@ -290,8 +300,10 @@ template <typename TranscriptParams> class BaseTranscript {
     {
         constexpr size_t num_challenges = sizeof...(Strings);
 
-        // Add challenge labels for current round to the manifest
-        manifest.add_challenge(round_number, labels...);
+        if (use_manifest) {
+            // Add challenge labels for current round to the manifest
+            manifest.add_challenge(round_number, labels...);
+        }
 
         // Compute the new challenge buffer from which we derive the challenges.
 
@@ -438,7 +450,13 @@ template <typename TranscriptParams> class BaseTranscript {
 
     [[nodiscard]] TranscriptManifest get_manifest() const { return manifest; };
 
-    void print() { manifest.print(); }
+    void print()
+    {
+        if (!use_manifest) {
+            info("Warning: manifest is not enabled!");
+        }
+        manifest.print();
+    }
 };
 
 template <typename Builder>

barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp

@@ -93,7 +93,7 @@ template <IsUltraFlavor Flavor> void DeciderProvingKey_<Flavor>::allocate_select
 template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_table_lookup_polynomials(const Circuit& circuit)
 {
-    PROFILE_THIS_NAME("allocate_table_lookup_polynomials_and_inverses");
+    PROFILE_THIS_NAME("allocate_table_lookup_and_lookup_read_polynomials");
 
     size_t table_offset = circuit.blocks.lookup.trace_offset;
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1193): can potentially improve memory footprint
@@ -143,6 +143,7 @@ template <IsUltraFlavor Flavor>
 void DeciderProvingKey_<Flavor>::allocate_databus_polynomials(const Circuit& circuit)
     requires HasDataBus<Flavor>
 {
+    PROFILE_THIS_NAME("allocate_databus_and_lookup_inverse_polynomials");
     proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);
     proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size);