feat: share the commitment key between instances to reduce mem (#8154)

Continuation of
#8118.

In AztecIVC (or ClientIVC), when we have multiple instances, we create a
commitment key for each one. However, since each of these instances are
the same size, there's no need to create a new one for each one.

When we're constructing an instance beyond the first one, we can reuse
the same commitment key from the AztecIVC accumulator, which saves
~123MB of memory for 2^17 circuits, a reduction of 15.6%.
<img width="1045" alt="After"
src="https://github.com/user-attachments/assets/032cf442-5c68-4c23-b4d2-16ab8c6812b7">


After the change, we cut down max memory by 123MB.
<img width="969" alt="Before"
src="https://github.com/user-attachments/assets/8e374ab5-8a4b-4395-964e-35e49fe8920a">

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.cpp

@@ -60,7 +60,13 @@ void AztecIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verifica
     circuit.add_recursive_proof(stdlib::recursion::init_default_agg_obj_indices<ClientCircuit>(circuit));
 
     // Construct the prover instance for circuit
-    auto prover_instance = std::make_shared<ProverInstance>(circuit, trace_structure);
+    std::shared_ptr<ProverInstance> prover_instance;
+    if (!initialized) {
+        prover_instance = std::make_shared<ProverInstance>(circuit, trace_structure);
+    } else {
+        prover_instance = std::make_shared<ProverInstance>(
+            circuit, trace_structure, fold_output.accumulator->proving_key.commitment_key);
+    }
 
     // Set the instance verification key from precomputed if available, else compute it
     instance_vk = precomputed_vk ? precomputed_vk : std::make_shared<VerificationKey>(prover_instance->proving_key);

barretenberg/cpp/src/barretenberg/aztec_ivc/aztec_ivc.test.cpp

@@ -132,6 +132,24 @@ TEST_F(AztecIVCTests, Basic)
     EXPECT_TRUE(ivc.prove_and_verify());
 };
 
+/**
+ * @brief A simple test demonstrating IVC for four mock circuits, which is slightly more than minimal.
+ * @details When accumulating only four circuits, we execute all the functionality of a full AztecIVC run.
+ *
+ */
+TEST_F(AztecIVCTests, BasicFour)
+{
+    AztecIVC ivc;
+
+    MockCircuitProducer circuit_producer;
+    for (size_t idx = 0; idx < 4; ++idx) {
+        Builder circuit = circuit_producer.create_next_circuit(ivc);
+        ivc.accumulate(circuit);
+    }
+
+    EXPECT_TRUE(ivc.prove_and_verify());
+};
+
 /**
  * @brief Check that the IVC fails to verify if an intermediate fold proof is invalid
  * @details When accumulating 4 circuits, there are 3 fold proofs to verify (the first two are recursively verfied and

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -37,7 +37,13 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verific
     circuit.add_recursive_proof(stdlib::recursion::init_default_agg_obj_indices<ClientCircuit>(circuit));
 
     // Construct the prover instance for circuit
-    auto prover_instance = std::make_shared<ProverInstance>(circuit, trace_structure);
+    std::shared_ptr<ProverInstance> prover_instance;
+    if (!initialized) {
+        prover_instance = std::make_shared<ProverInstance>(circuit, trace_structure);
+    } else {
+        prover_instance = std::make_shared<ProverInstance>(
+            circuit, trace_structure, fold_output.accumulator->proving_key.commitment_key);
+    }
 
     // Track the maximum size of each block for all circuits porcessed (for debugging purposes only)
     max_block_size_tracker.update(circuit);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -66,7 +66,6 @@ class ClientIVC {
   public:
     GoblinProver goblin;
     ProverFoldOutput fold_output;
-    std::shared_ptr<ProverInstance> prover_accumulator;
     std::shared_ptr<VerifierInstance> verifier_accumulator;
     std::shared_ptr<VerificationKey> instance_vk;
 

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp

@@ -90,7 +90,7 @@ TEST_F(ClientIVCTests, Basic)
 };
 
 /**
- * @brief A simple-as-possible test demonstrating IVC for three mock circuits
+ * @brief A simple test demonstrating IVC for three mock circuits which does more logic than just two circuits.
  *
  */
 TEST_F(ClientIVCTests, BasicThree)

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -124,11 +124,16 @@ template <typename FF, typename CommitmentKey_> class ProvingKey_ {
     std::vector<FF> public_inputs;
 
     ProvingKey_() = default;
-    ProvingKey_(const size_t circuit_size, const size_t num_public_inputs)
+    ProvingKey_(const size_t circuit_size,
+                const size_t num_public_inputs,
+                std::shared_ptr<CommitmentKey_> commitment_key = nullptr)
     {
-        {
+        if (commitment_key == nullptr) {
             ZoneScopedN("init commitment key");
             this->commitment_key = std::make_shared<CommitmentKey_>(circuit_size + 1);
+        } else {
+            // Don't create another commitment key if we already have one
+            this->commitment_key = commitment_key;
         }
         this->evaluation_domain = bb::EvaluationDomain<FF>(circuit_size, circuit_size);
         this->circuit_size = circuit_size;

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mega_flavor.hpp

@@ -412,12 +412,13 @@ class MegaFlavor {
      */
     class ProvingKey : public ProvingKey_<FF, CommitmentKey> {
       public:
-        // Expose constructors on the base class
         using Base = ProvingKey_<FF, CommitmentKey>;
-        using Base::Base;
 
-        ProvingKey(const size_t circuit_size, const size_t num_public_inputs)
-            : Base(circuit_size, num_public_inputs)
+        ProvingKey() = default;
+        ProvingKey(const size_t circuit_size,
+                   const size_t num_public_inputs,
+                   std::shared_ptr<CommitmentKey> commitment_key = nullptr)
+            : Base(circuit_size, num_public_inputs, commitment_key)
             , polynomials(circuit_size){};
 
         std::vector<uint32_t> memory_read_records;

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_flavor.hpp

@@ -331,12 +331,13 @@ class UltraFlavor {
      */
     class ProvingKey : public ProvingKey_<FF, CommitmentKey> {
       public:
-        // Expose constructors on the base class
         using Base = ProvingKey_<FF, CommitmentKey>;
-        using Base::Base;
 
-        ProvingKey(const size_t circuit_size, const size_t num_public_inputs)
-            : Base(circuit_size, num_public_inputs)
+        ProvingKey() = default;
+        ProvingKey(const size_t circuit_size,
+                   const size_t num_public_inputs,
+                   std::shared_ptr<CommitmentKey> commitment_key = nullptr)
+            : Base(circuit_size, num_public_inputs, std::move(commitment_key))
             , polynomials(circuit_size){};
 
         std::vector<uint32_t> memory_read_records;

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_keccak.hpp

@@ -278,12 +278,13 @@ class UltraKeccakFlavor {
      */
     class ProvingKey : public ProvingKey_<FF, CommitmentKey> {
       public:
-        // Expose constructors on the base class
         using Base = ProvingKey_<FF, CommitmentKey>;
-        using Base::Base;
 
-        ProvingKey(const size_t circuit_size, const size_t num_public_inputs)
-            : Base(circuit_size, num_public_inputs)
+        ProvingKey() = default;
+        ProvingKey(const size_t circuit_size,
+                   const size_t num_public_inputs,
+                   std::shared_ptr<CommitmentKey> commitment_key = nullptr)
+            : Base(circuit_size, num_public_inputs, commitment_key)
             , polynomials(circuit_size){};
 
         std::vector<uint32_t> memory_read_records;

barretenberg/cpp/src/barretenberg/sumcheck/instance/prover_instance.hpp

@@ -44,11 +44,14 @@ template <class Flavor> class ProverInstance_ {
     std::vector<FF> gate_challenges;
     FF target_sum;
 
-    ProverInstance_(Circuit& circuit, TraceStructure trace_structure = TraceStructure::NONE)
+    ProverInstance_(Circuit& circuit,
+                    TraceStructure trace_structure = TraceStructure::NONE,
+                    std::shared_ptr<typename Flavor::CommitmentKey> commitment_key = nullptr)
     {
         BB_OP_COUNT_TIME_NAME("ProverInstance(Circuit&)");
         circuit.add_gates_to_ensure_all_polys_are_non_zero();
         circuit.finalize_circuit();
+        info("finalized gate count: ", circuit.num_gates);
 
         // Set flag indicating whether the polynomials will be constructed with fixed block sizes for each gate type
         const bool is_structured = (trace_structure != TraceStructure::NONE);
@@ -70,7 +73,7 @@ template <class Flavor> class ProverInstance_ {
         }
         {
             ZoneScopedN("constructing proving key");
-            proving_key = ProvingKey(dyadic_circuit_size, circuit.public_inputs.size());
+            proving_key = ProvingKey(dyadic_circuit_size, circuit.public_inputs.size(), commitment_key);
         }
 
         // Construct and add to proving key the wire, selector and copy constraint polynomials