From 653ed9669a35451c05a36ae671d841e35c7e11f2 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 4 Nov 2024 22:26:16 +0000 Subject: [PATCH 01/25] basic mega honk tests passes with block overflow --- .../client_ivc/client_ivc.test.cpp | 26 +- .../cpp/src/barretenberg/constants.hpp | 6 +- .../execution_trace/execution_trace.cpp | 2 + .../arithmetization/arithmetization.hpp | 2 +- .../arithmetization/mega_arithmetization.hpp | 95 ++++- .../arithmetization/ultra_arithmetization.hpp | 3 + .../ultra_honk/decider_proving_key.hpp | 326 +++++++++--------- .../ultra_honk/mega_composer.test.cpp | 22 ++ 8 files changed, 315 insertions(+), 167 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index c7e85ed675f..370e7015478 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -386,4 +386,28 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) verify_ivc(proof, ivc); }; ASSERT_NO_FATAL_FAILURE(run_test()); -} \ No newline at end of file +} + +// /** +// * @brief Test use of miscellaneous block +// * +// */ +// TEST_F(ClientIVCTests, MiscellaneousBlock) +// { +// ClientIVC ivc; +// ivc.trace_structure = TraceStructure::TINY_TEST; + +// MockCircuitProducer circuit_producer; + +// size_t NUM_CIRCUITS = 2; + +// // Construct and accumulate some circuits of varying size +// size_t log2_num_gates = 5; +// for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { +// auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates); +// ivc.accumulate(circuit); +// log2_num_gates += 2; +// } + +// EXPECT_TRUE(ivc.prove_and_verify()); +// }; \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/constants.hpp b/barretenberg/cpp/src/barretenberg/constants.hpp index bdfa57e5f95..8c052f57ee7 100644 --- a/barretenberg/cpp/src/barretenberg/constants.hpp +++ b/barretenberg/cpp/src/barretenberg/constants.hpp @@ -11,7 +11,9 @@ static constexpr uint32_t CONST_PROOF_SIZE_LOG_N = 28; // circuits being folded. static constexpr uint32_t CONST_PG_LOG_N = 20; -static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 70000; +static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 8; // DEBUG! +// static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 70000; -static constexpr uint32_t MAX_DATABUS_SIZE = 10000; +static constexpr uint32_t MAX_DATABUS_SIZE = 3; // DEBUG! +// static constexpr uint32_t MAX_DATABUS_SIZE = 10000; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index 93945d1907e..15bbe804221 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -134,6 +134,8 @@ typename ExecutionTrace_::TraceData ExecutionTrace_::construct_t // Store the offset of the block containing RAM/ROM read/write gates for use in updating memory records if (block.has_ram_rom) { + // WORKTODO: uh oh. is this a problem if now the misc block can contain aux stuff too? what updates are + // needed? trace_data.ram_rom_offset = offset; } // Store offset of public inputs block for use in the pub input mechanism of the permutation argument diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index b5408341edb..33d28c664ed 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -30,7 +30,7 @@ struct StackTraces { // A set of fixed block size conigurations to be used with the structured execution trace. The actual block sizes // corresponding to these settings are defined in the corresponding arithmetization classes (Ultra/Mega). For efficiency // it is best to use the smallest possible block sizes to accommodate a given situation. -enum class TraceStructure { NONE, SMALL_TEST, CLIENT_IVC_BENCH, E2E_FULL_TEST }; +enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_FULL_TEST }; /** * @brief Basic structure for storing gate data in a builder diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index 67a77dfa715..d84d00052d8 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -31,13 +31,14 @@ template class MegaArith { T poseidon2_external; T poseidon2_internal; T lookup; + T miscellaneous; auto get() { return RefArray{ ecc_op, pub_inputs, busread, arithmetic, delta_range, elliptic, aux, poseidon2_external, poseidon2_internal, - lookup }; + lookup, miscellaneous }; } auto get_gate_blocks() @@ -50,6 +51,47 @@ template class MegaArith { }; private: + // A tiny structuring (for testing without recursive verifications only) + struct TinyTestStructuredBlockSizes : public MegaTraceBlocks { + TinyTestStructuredBlockSizes() + { + // const uint32_t FIXED_SIZE = 1 << 14; + // this->ecc_op = FIXED_SIZE; + // this->pub_inputs = FIXED_SIZE; + // this->busread = FIXED_SIZE; + // this->arithmetic = 1 << 15; + // this->delta_range = FIXED_SIZE; + // this->elliptic = FIXED_SIZE; + // this->aux = FIXED_SIZE; + // this->poseidon2_external = FIXED_SIZE; + // this->poseidon2_internal = 1 << 15; + // this->lookup = FIXED_SIZE; + // this->miscellaneous = 0; + + // this->ecc_op = 50; + // this->pub_inputs = 33; + // this->busread = 3; + // this->arithmetic = 3417; // 3418 + // this->delta_range = 2002; + // this->elliptic = 2; + // this->aux = 26; + // this->poseidon2_external = 812; + // this->poseidon2_internal = 4619; + // this->lookup = 2; + + this->ecc_op = 18; + this->pub_inputs = 1; + this->busread = 3; + this->arithmetic = 7; // 8 + this->delta_range = 2; + this->elliptic = 2; + this->aux = 2; + this->poseidon2_external = 2; + this->poseidon2_internal = 2; + this->lookup = 2; + this->miscellaneous = 0; + } + }; // An arbitrary but small-ish structuring that can be used for generic unit testing with non-trivial circuits struct SmallTestStructuredBlockSizes : public MegaTraceBlocks { SmallTestStructuredBlockSizes() @@ -65,6 +107,7 @@ template class MegaArith { this->poseidon2_external = FIXED_SIZE; this->poseidon2_internal = 1 << 15; this->lookup = FIXED_SIZE; + this->miscellaneous = 0; } }; @@ -82,6 +125,7 @@ template class MegaArith { this->busread = 1 << 7; this->poseidon2_external = 2500; this->poseidon2_internal = 14000; + this->miscellaneous = 0; } }; @@ -99,6 +143,7 @@ template class MegaArith { this->poseidon2_external = 30128; this->poseidon2_internal = 172000; this->lookup = 200000; + this->miscellaneous = 0; } }; @@ -161,6 +206,8 @@ template class MegaArith { struct TraceBlocks : public MegaTraceBlocks { + bool has_overflow = false; + TraceBlocks() { this->aux.has_ram_rom = true; @@ -175,6 +222,9 @@ template class MegaArith { switch (setting) { case TraceStructure::NONE: break; + case TraceStructure::TINY_TEST: + fixed_block_sizes = TinyTestStructuredBlockSizes(); + break; case TraceStructure::SMALL_TEST: fixed_block_sizes = SmallTestStructuredBlockSizes(); break; @@ -212,6 +262,7 @@ template class MegaArith { info("poseidon ext :\t", this->poseidon2_external.size(), "/", this->poseidon2_external.get_fixed_size()); info("poseidon int :\t", this->poseidon2_internal.size(), "/", this->poseidon2_internal.get_fixed_size()); info("lookups :\t", this->lookup.size(), "/", this->lookup.get_fixed_size()); + info("miscellaneous :\t", this->miscellaneous.size(), "/", this->miscellaneous.get_fixed_size()); info(""); } @@ -226,15 +277,51 @@ template class MegaArith { void check_within_fixed_sizes() { - for (auto block : this->get()) { - if (block.size() > block.get_fixed_size()) { + using SelectorType = SlabVector; + using WireType = SlabVector; + + for (auto& block : this->get()) { + size_t block_size = block.size(); + size_t fixed_block_size = block.get_fixed_size(); + if (block_size > block.get_fixed_size() && block != this->miscellaneous) { + // Set flag indicating that at least one block exceeds capacity and the misc block is in use + this->has_overflow = true; + + // move the excess wire and selector data from the offending block to the miscellaneous block + for (auto [wire, misc_wire] : zip_view(block.wires, this->miscellaneous.wires)) { + WireType overflow(wire.begin() + static_cast(fixed_block_size), wire.end()); + // misc_wire.push_back(overflow); + for (const auto& val : overflow) { + misc_wire.emplace_back(val); + } + wire.resize(fixed_block_size); + } + for (auto [selector, misc_selector] : zip_view(block.selectors, this->miscellaneous.selectors)) { + SelectorType overflow(selector.begin() + static_cast(fixed_block_size), + selector.end()); + // misc_selector.push_back(std::move(overflow)); + for (const auto& val : overflow) { + misc_selector.emplace_back(val); + } + selector.resize(fixed_block_size); + } + + this->miscellaneous.has_ram_rom = block.has_ram_rom; + this->miscellaneous.is_pub_inputs = block.is_pub_inputs; + this->summarize(); + info(block.size()); info("WARNING: Num gates in circuit block exceeds the specified fixed size - execution trace will " "not be constructed correctly!"); summarize(); - ASSERT(false); + // ASSERT(false); } } + // WORKTODO: for now we dont handle this + ASSERT(!this->miscellaneous.has_ram_rom); + ASSERT(!this->miscellaneous.is_pub_inputs); + // Set the fixed size of the miscellaneous block to its current size + this->miscellaneous.set_fixed_size(static_cast(this->miscellaneous.size())); } bool operator==(const TraceBlocks& other) const = default; diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index be1e47fac17..349b41bccec 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -95,6 +95,8 @@ template class UltraArith { struct TraceBlocks : public UltraTraceBlocks { + bool has_overflow = false; + TraceBlocks() { this->aux.has_ram_rom = true; @@ -110,6 +112,7 @@ template class UltraArith { case TraceStructure::NONE: break; // We don't use Ultra in ClientIvc so no need for anything other than sizing for simple unit tests + case TraceStructure::TINY_TEST: case TraceStructure::SMALL_TEST: case TraceStructure::CLIENT_IVC_BENCH: case TraceStructure::E2E_FULL_TEST: diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 6ccd39446c2..c9dd720f059 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -61,7 +61,9 @@ template class DeciderProvingKey_ { // If using a structured trace, set fixed block sizes, check their validity, and set the dyadic circuit size if (is_structured) { circuit.blocks.set_fixed_block_sizes(trace_structure); // set the fixed sizes for each block - circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size + circuit.blocks.summarize(); + circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size + circuit.blocks.summarize(); dyadic_circuit_size = compute_structured_dyadic_size(circuit); // set the dyadic size accordingly } else { dyadic_circuit_size = compute_dyadic_size(circuit); // set dyadic size directly from circuit block sizes @@ -88,164 +90,12 @@ template class DeciderProvingKey_ { if (IsGoblinFlavor && !is_structured) { // Allocate full size polynomials proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); - } else { // Allocate only a correct amount of memory for each polynomial - // Allocate the wires and selectors polynomials - { - PROFILE_THIS_NAME("allocating wires"); - vinfo("allocating wires"); - - for (auto& wire : proving_key.polynomials.get_wires()) { - wire = Polynomial::shiftable(proving_key.circuit_size); - } - } - { - PROFILE_THIS_NAME("allocating gate selectors"); - vinfo("allocating gate selectors"); - - // Define gate selectors over the block they are isolated to - for (auto [selector, block] : - zip_view(proving_key.polynomials.get_gate_selectors(), circuit.blocks.get_gate_blocks())) { - - // TODO(https://github.com/AztecProtocol/barretenberg/issues/914): q_arith is currently used - // in aux block. - if (&block == &circuit.blocks.arithmetic) { - size_t arith_size = circuit.blocks.aux.trace_offset - - circuit.blocks.arithmetic.trace_offset + - circuit.blocks.aux.get_fixed_size(is_structured); - selector = Polynomial( - arith_size, proving_key.circuit_size, circuit.blocks.arithmetic.trace_offset); - } else { - selector = Polynomial( - block.get_fixed_size(is_structured), proving_key.circuit_size, block.trace_offset); - } - } - } - { - PROFILE_THIS_NAME("allocating non-gate selectors"); - vinfo("allocating non-gate selectors"); - - // Set the other non-gate selector polynomials to full size - for (auto& selector : proving_key.polynomials.get_non_gate_selectors()) { - selector = Polynomial(proving_key.circuit_size); - } - } - if constexpr (IsGoblinFlavor) { - PROFILE_THIS_NAME("allocating ecc op wires and selector"); - vinfo("allocating ecc op wires and selector"); - - // Allocate the ecc op wires and selector - const size_t ecc_op_block_size = circuit.blocks.ecc_op.get_fixed_size(is_structured); - const size_t op_wire_offset = Flavor::has_zero_row ? 1 : 0; - for (auto& wire : proving_key.polynomials.get_ecc_op_wires()) { - wire = Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); - } - proving_key.polynomials.lagrange_ecc_op = - Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); - } - - if constexpr (HasDataBus) { - proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.calldata_read_counts = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata_read_counts = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata_read_tags = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data_read_counts = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data_read_tags = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1107): Restricting databus_id to - // databus_size leads to failure. - // const size_t databus_size = std::max({ calldata.size(), secondary_calldata.size(), - // return_data.size() }); - proving_key.polynomials.databus_id = Polynomial(proving_key.circuit_size, proving_key.circuit_size); - } - const size_t max_tables_size = - std::min(static_cast(MAX_LOOKUP_TABLES_SIZE), dyadic_circuit_size - 1); - size_t table_offset = dyadic_circuit_size - max_tables_size; - { - PROFILE_THIS_NAME("allocating table polynomials"); - vinfo("allocating table polynomials"); - - ASSERT(dyadic_circuit_size > max_tables_size); - - // Allocate the table polynomials - if constexpr (IsHonkFlavor) { - for (auto& poly : proving_key.polynomials.get_tables()) { - poly = typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - } - } - } - { - PROFILE_THIS_NAME("allocating sigmas and ids"); - vinfo("allocating sigmas and ids"); - - for (auto& sigma : proving_key.polynomials.get_sigmas()) { - sigma = typename Flavor::Polynomial(proving_key.circuit_size); - } - for (auto& id : proving_key.polynomials.get_ids()) { - id = typename Flavor::Polynomial(proving_key.circuit_size); - } - } - { - ZoneScopedN("allocating lookup read counts and tags"); - // Allocate the read counts and tags polynomials - vinfo("allocating lookup read counts and tags"); - proving_key.polynomials.lookup_read_counts = - typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - proving_key.polynomials.lookup_read_tags = - typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - } - { - ZoneScopedN("allocating lookup and databus inverses"); - // Allocate the lookup_inverses polynomial - vinfo("allocating lookup and databus inverses"); - const size_t lookup_offset = static_cast(circuit.blocks.lookup.trace_offset); - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts - // at top of trace - const size_t table_offset = - dyadic_circuit_size - - std::min(dyadic_circuit_size - 1, static_cast(MAX_LOOKUP_TABLES_SIZE)); - const size_t lookup_inverses_start = std::min(lookup_offset, table_offset); - const size_t lookup_inverses_end = - std::min(dyadic_circuit_size, - std::max(lookup_offset + circuit.blocks.lookup.get_fixed_size(is_structured), - table_offset + MAX_LOOKUP_TABLES_SIZE)); - proving_key.polynomials.lookup_inverses = Polynomial( - lookup_inverses_end - lookup_inverses_start, dyadic_circuit_size, lookup_inverses_start); - if constexpr (HasDataBus) { - const size_t q_busread_end = - circuit.blocks.busread.trace_offset + circuit.blocks.busread.get_fixed_size(is_structured); - // Allocate the databus inverse polynomials - proving_key.polynomials.calldata_inverses = - Polynomial(std::max(circuit.get_calldata().size(), q_busread_end), dyadic_circuit_size); - proving_key.polynomials.secondary_calldata_inverses = Polynomial( - std::max(circuit.get_secondary_calldata().size(), q_busread_end), dyadic_circuit_size); - proving_key.polynomials.return_data_inverses = - Polynomial(std::max(circuit.get_return_data().size(), q_busread_end), dyadic_circuit_size); - } - } - { - PROFILE_THIS_NAME("constructing z_perm"); - vinfo("constructing z_perm"); - - // Allocate the z_perm polynomial - proving_key.polynomials.z_perm = Polynomial::shiftable(proving_key.circuit_size); - } - - { - PROFILE_THIS_NAME("allocating lagrange polynomials"); - vinfo("allocating lagrange polynomials"); - - // First and last lagrange polynomials (in the full circuit size) - proving_key.polynomials.lagrange_first = Polynomial(1, dyadic_circuit_size, 0); - proving_key.polynomials.lagrange_last = Polynomial(1, dyadic_circuit_size, dyadic_circuit_size - 1); - } + } + // WORKTODO: in the overflow case, just allocate full polys for simplicity + else if (is_structured && circuit.blocks.has_overflow) { + proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); + } else { + instantiate_polynomials_based_on_structured_trace(circuit); } // We can finally set the shifted polynomials now that all of the to_be_shifted polynomials are // defined. @@ -328,6 +178,164 @@ template class DeciderProvingKey_ { void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; + + void instantiate_polynomials_based_on_structured_trace(Circuit& circuit) + { + // Allocate the wires and selectors polynomials + { + PROFILE_THIS_NAME("allocating wires"); + vinfo("allocating wires"); + + for (auto& wire : proving_key.polynomials.get_wires()) { + wire = Polynomial::shiftable(proving_key.circuit_size); + } + } + { + PROFILE_THIS_NAME("allocating gate selectors"); + vinfo("allocating gate selectors"); + + // Define gate selectors over the block they are isolated to + for (auto [selector, block] : + zip_view(proving_key.polynomials.get_gate_selectors(), circuit.blocks.get_gate_blocks())) { + + // TODO(https://github.com/AztecProtocol/barretenberg/issues/914): q_arith is currently used + // in aux block. + if (&block == &circuit.blocks.arithmetic) { + size_t arith_size = circuit.blocks.aux.trace_offset - circuit.blocks.arithmetic.trace_offset + + circuit.blocks.aux.get_fixed_size(is_structured); + selector = Polynomial(arith_size, proving_key.circuit_size, circuit.blocks.arithmetic.trace_offset); + } else { + selector = + Polynomial(block.get_fixed_size(is_structured), proving_key.circuit_size, block.trace_offset); + } + } + } + { + PROFILE_THIS_NAME("allocating non-gate selectors"); + vinfo("allocating non-gate selectors"); + + // Set the other non-gate selector polynomials to full size + for (auto& selector : proving_key.polynomials.get_non_gate_selectors()) { + selector = Polynomial(proving_key.circuit_size); + } + } + if constexpr (IsGoblinFlavor) { + PROFILE_THIS_NAME("allocating ecc op wires and selector"); + vinfo("allocating ecc op wires and selector"); + + // Allocate the ecc op wires and selector + const size_t ecc_op_block_size = circuit.blocks.ecc_op.get_fixed_size(is_structured); + const size_t op_wire_offset = Flavor::has_zero_row ? 1 : 0; + for (auto& wire : proving_key.polynomials.get_ecc_op_wires()) { + wire = Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); + } + proving_key.polynomials.lagrange_ecc_op = + Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); + } + + if constexpr (HasDataBus) { + // WORKTODO: could be this: + // for (auto& poly : proving_key.polynomials.get_databus_entities()) { + // poly = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + // } + proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata_read_counts = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata_read_tags = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1107): Restricting databus_id to + // databus_size leads to failure. + // const size_t databus_size = std::max({ calldata.size(), secondary_calldata.size(), + // return_data.size() }); + proving_key.polynomials.databus_id = Polynomial(proving_key.circuit_size, proving_key.circuit_size); + } + const size_t max_tables_size = std::min(static_cast(MAX_LOOKUP_TABLES_SIZE), dyadic_circuit_size - 1); + size_t table_offset = dyadic_circuit_size - max_tables_size; + { + PROFILE_THIS_NAME("allocating table polynomials"); + vinfo("allocating table polynomials"); + + ASSERT(dyadic_circuit_size > max_tables_size); + + // Allocate the table polynomials + if constexpr (IsHonkFlavor) { + for (auto& poly : proving_key.polynomials.get_tables()) { + poly = typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + } + } + } + { + PROFILE_THIS_NAME("allocating sigmas and ids"); + vinfo("allocating sigmas and ids"); + + for (auto& sigma : proving_key.polynomials.get_sigmas()) { + sigma = typename Flavor::Polynomial(proving_key.circuit_size); + } + for (auto& id : proving_key.polynomials.get_ids()) { + id = typename Flavor::Polynomial(proving_key.circuit_size); + } + } + { + ZoneScopedN("allocating lookup read counts and tags"); + // Allocate the read counts and tags polynomials + vinfo("allocating lookup read counts and tags"); + proving_key.polynomials.lookup_read_counts = + typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + proving_key.polynomials.lookup_read_tags = + typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + } + { + ZoneScopedN("allocating lookup and databus inverses"); + // Allocate the lookup_inverses polynomial + vinfo("allocating lookup and databus inverses"); + const size_t lookup_offset = static_cast(circuit.blocks.lookup.trace_offset); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts + // at top of trace + const size_t table_offset = + dyadic_circuit_size - std::min(dyadic_circuit_size - 1, static_cast(MAX_LOOKUP_TABLES_SIZE)); + const size_t lookup_inverses_start = std::min(lookup_offset, table_offset); + const size_t lookup_inverses_end = + std::min(dyadic_circuit_size, + std::max(lookup_offset + circuit.blocks.lookup.get_fixed_size(is_structured), + table_offset + MAX_LOOKUP_TABLES_SIZE)); + proving_key.polynomials.lookup_inverses = + Polynomial(lookup_inverses_end - lookup_inverses_start, dyadic_circuit_size, lookup_inverses_start); + if constexpr (HasDataBus) { + const size_t q_busread_end = + circuit.blocks.busread.trace_offset + circuit.blocks.busread.get_fixed_size(is_structured); + // Allocate the databus inverse polynomials + proving_key.polynomials.calldata_inverses = + Polynomial(std::max(circuit.get_calldata().size(), q_busread_end), dyadic_circuit_size); + proving_key.polynomials.secondary_calldata_inverses = + Polynomial(std::max(circuit.get_secondary_calldata().size(), q_busread_end), dyadic_circuit_size); + proving_key.polynomials.return_data_inverses = + Polynomial(std::max(circuit.get_return_data().size(), q_busread_end), dyadic_circuit_size); + } + } + { + PROFILE_THIS_NAME("constructing z_perm"); + vinfo("constructing z_perm"); + + // Allocate the z_perm polynomial + proving_key.polynomials.z_perm = Polynomial::shiftable(proving_key.circuit_size); + } + + { + PROFILE_THIS_NAME("allocating lagrange polynomials"); + vinfo("allocating lagrange polynomials"); + + // First and last lagrange polynomials (in the full circuit size) + proving_key.polynomials.lagrange_first = Polynomial(1, dyadic_circuit_size, 0); + proving_key.polynomials.lagrange_last = Polynomial(1, dyadic_circuit_size, dyadic_circuit_size - 1); + } + } }; } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index 2bbac0e9a54..b3ed22c903a 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -209,3 +209,25 @@ TEST_F(MegaHonkTests, MultipleCircuitsHonkAndMerge) EXPECT_EQ(result, expected); } } + +/** + * @brief Test proof construction/verification for a structured execution trace + * + */ +TEST_F(MegaHonkTests, MiscellaneousBlock) +{ + MegaCircuitBuilder builder; + + GoblinMockCircuits::construct_simple_circuit(builder); + + // Construct and verify Honk proof using a structured trace + TraceStructure trace_structure = TraceStructure::TINY_TEST; + auto proving_key = std::make_shared>(builder, trace_structure); + // builder.blocks.summarize(); + MegaProver prover(proving_key); + // builder.blocks.summarize(); + auto verification_key = std::make_shared(proving_key->proving_key); + MegaVerifier verifier(verification_key); + auto proof = prover.construct_proof(); + EXPECT_TRUE(verifier.verify_proof(proof)); +} From 07c8f0ce3dab234e687ae596716aeaf2639162f4 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 5 Nov 2024 15:45:06 +0000 Subject: [PATCH 02/25] reinstate lookup/databus constants --- .../cpp/src/barretenberg/constants.hpp | 8 +- .../ultra_honk/decider_proving_key.hpp | 324 +++++++++--------- 2 files changed, 167 insertions(+), 165 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/constants.hpp b/barretenberg/cpp/src/barretenberg/constants.hpp index 8c052f57ee7..6cbebf82cd0 100644 --- a/barretenberg/cpp/src/barretenberg/constants.hpp +++ b/barretenberg/cpp/src/barretenberg/constants.hpp @@ -11,9 +11,9 @@ static constexpr uint32_t CONST_PROOF_SIZE_LOG_N = 28; // circuits being folded. static constexpr uint32_t CONST_PG_LOG_N = 20; -static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 8; // DEBUG! -// static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 70000; +// static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 8; // DEBUG! +static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 70000; -static constexpr uint32_t MAX_DATABUS_SIZE = 3; // DEBUG! -// static constexpr uint32_t MAX_DATABUS_SIZE = 10000; +// static constexpr uint32_t MAX_DATABUS_SIZE = 3; // DEBUG! +static constexpr uint32_t MAX_DATABUS_SIZE = 10000; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index c9dd720f059..045e5c3ae9b 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -91,11 +91,171 @@ template class DeciderProvingKey_ { // Allocate full size polynomials proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); } - // WORKTODO: in the overflow case, just allocate full polys for simplicity + // If using structured trace but overflow has occurred (misc. block in use), allocate full size polynomials else if (is_structured && circuit.blocks.has_overflow) { proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); - } else { - instantiate_polynomials_based_on_structured_trace(circuit); + } else { // Allocate only a correct amount of memory for each polynomial + // Allocate the wires and selectors polynomials + { + PROFILE_THIS_NAME("allocating wires"); + vinfo("allocating wires"); + + for (auto& wire : proving_key.polynomials.get_wires()) { + wire = Polynomial::shiftable(proving_key.circuit_size); + } + } + { + PROFILE_THIS_NAME("allocating gate selectors"); + vinfo("allocating gate selectors"); + + // Define gate selectors over the block they are isolated to + for (auto [selector, block] : + zip_view(proving_key.polynomials.get_gate_selectors(), circuit.blocks.get_gate_blocks())) { + + // TODO(https://github.com/AztecProtocol/barretenberg/issues/914): q_arith is currently used + // in aux block. + if (&block == &circuit.blocks.arithmetic) { + size_t arith_size = circuit.blocks.aux.trace_offset - + circuit.blocks.arithmetic.trace_offset + + circuit.blocks.aux.get_fixed_size(is_structured); + selector = Polynomial( + arith_size, proving_key.circuit_size, circuit.blocks.arithmetic.trace_offset); + } else { + selector = Polynomial( + block.get_fixed_size(is_structured), proving_key.circuit_size, block.trace_offset); + } + } + } + { + PROFILE_THIS_NAME("allocating non-gate selectors"); + vinfo("allocating non-gate selectors"); + + // Set the other non-gate selector polynomials to full size + for (auto& selector : proving_key.polynomials.get_non_gate_selectors()) { + selector = Polynomial(proving_key.circuit_size); + } + } + if constexpr (IsGoblinFlavor) { + PROFILE_THIS_NAME("allocating ecc op wires and selector"); + vinfo("allocating ecc op wires and selector"); + + // Allocate the ecc op wires and selector + const size_t ecc_op_block_size = circuit.blocks.ecc_op.get_fixed_size(is_structured); + const size_t op_wire_offset = Flavor::has_zero_row ? 1 : 0; + for (auto& wire : proving_key.polynomials.get_ecc_op_wires()) { + wire = Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); + } + proving_key.polynomials.lagrange_ecc_op = + Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); + } + + if constexpr (HasDataBus) { + // WORKTODO: could be this: + // for (auto& poly : proving_key.polynomials.get_databus_entities()) { + // poly = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + // } + proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.calldata_read_counts = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata_read_counts = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.secondary_calldata_read_tags = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data_read_counts = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + proving_key.polynomials.return_data_read_tags = + Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); + + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1107): Restricting databus_id to + // databus_size leads to failure. + // const size_t databus_size = std::max({ calldata.size(), secondary_calldata.size(), + // return_data.size() }); + proving_key.polynomials.databus_id = Polynomial(proving_key.circuit_size, proving_key.circuit_size); + } + const size_t max_tables_size = + std::min(static_cast(MAX_LOOKUP_TABLES_SIZE), dyadic_circuit_size - 1); + size_t table_offset = dyadic_circuit_size - max_tables_size; + { + PROFILE_THIS_NAME("allocating table polynomials"); + vinfo("allocating table polynomials"); + + ASSERT(dyadic_circuit_size > max_tables_size); + + // Allocate the table polynomials + if constexpr (IsHonkFlavor) { + for (auto& poly : proving_key.polynomials.get_tables()) { + poly = typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + } + } + } + { + PROFILE_THIS_NAME("allocating sigmas and ids"); + vinfo("allocating sigmas and ids"); + + for (auto& sigma : proving_key.polynomials.get_sigmas()) { + sigma = typename Flavor::Polynomial(proving_key.circuit_size); + } + for (auto& id : proving_key.polynomials.get_ids()) { + id = typename Flavor::Polynomial(proving_key.circuit_size); + } + } + { + ZoneScopedN("allocating lookup read counts and tags"); + // Allocate the read counts and tags polynomials + vinfo("allocating lookup read counts and tags"); + proving_key.polynomials.lookup_read_counts = + typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + proving_key.polynomials.lookup_read_tags = + typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); + } + { + ZoneScopedN("allocating lookup and databus inverses"); + // Allocate the lookup_inverses polynomial + vinfo("allocating lookup and databus inverses"); + const size_t lookup_offset = static_cast(circuit.blocks.lookup.trace_offset); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts + // at top of trace + const size_t table_offset = + dyadic_circuit_size - + std::min(dyadic_circuit_size - 1, static_cast(MAX_LOOKUP_TABLES_SIZE)); + const size_t lookup_inverses_start = std::min(lookup_offset, table_offset); + const size_t lookup_inverses_end = + std::min(dyadic_circuit_size, + std::max(lookup_offset + circuit.blocks.lookup.get_fixed_size(is_structured), + table_offset + MAX_LOOKUP_TABLES_SIZE)); + proving_key.polynomials.lookup_inverses = Polynomial( + lookup_inverses_end - lookup_inverses_start, dyadic_circuit_size, lookup_inverses_start); + if constexpr (HasDataBus) { + const size_t q_busread_end = + circuit.blocks.busread.trace_offset + circuit.blocks.busread.get_fixed_size(is_structured); + // Allocate the databus inverse polynomials + proving_key.polynomials.calldata_inverses = + Polynomial(std::max(circuit.get_calldata().size(), q_busread_end), dyadic_circuit_size); + proving_key.polynomials.secondary_calldata_inverses = Polynomial( + std::max(circuit.get_secondary_calldata().size(), q_busread_end), dyadic_circuit_size); + proving_key.polynomials.return_data_inverses = + Polynomial(std::max(circuit.get_return_data().size(), q_busread_end), dyadic_circuit_size); + } + } + { + PROFILE_THIS_NAME("constructing z_perm"); + vinfo("constructing z_perm"); + + // Allocate the z_perm polynomial + proving_key.polynomials.z_perm = Polynomial::shiftable(proving_key.circuit_size); + } + + { + PROFILE_THIS_NAME("allocating lagrange polynomials"); + vinfo("allocating lagrange polynomials"); + + // First and last lagrange polynomials (in the full circuit size) + proving_key.polynomials.lagrange_first = Polynomial(1, dyadic_circuit_size, 0); + proving_key.polynomials.lagrange_last = Polynomial(1, dyadic_circuit_size, dyadic_circuit_size - 1); + } } // We can finally set the shifted polynomials now that all of the to_be_shifted polynomials are // defined. @@ -178,164 +338,6 @@ template class DeciderProvingKey_ { void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; - - void instantiate_polynomials_based_on_structured_trace(Circuit& circuit) - { - // Allocate the wires and selectors polynomials - { - PROFILE_THIS_NAME("allocating wires"); - vinfo("allocating wires"); - - for (auto& wire : proving_key.polynomials.get_wires()) { - wire = Polynomial::shiftable(proving_key.circuit_size); - } - } - { - PROFILE_THIS_NAME("allocating gate selectors"); - vinfo("allocating gate selectors"); - - // Define gate selectors over the block they are isolated to - for (auto [selector, block] : - zip_view(proving_key.polynomials.get_gate_selectors(), circuit.blocks.get_gate_blocks())) { - - // TODO(https://github.com/AztecProtocol/barretenberg/issues/914): q_arith is currently used - // in aux block. - if (&block == &circuit.blocks.arithmetic) { - size_t arith_size = circuit.blocks.aux.trace_offset - circuit.blocks.arithmetic.trace_offset + - circuit.blocks.aux.get_fixed_size(is_structured); - selector = Polynomial(arith_size, proving_key.circuit_size, circuit.blocks.arithmetic.trace_offset); - } else { - selector = - Polynomial(block.get_fixed_size(is_structured), proving_key.circuit_size, block.trace_offset); - } - } - } - { - PROFILE_THIS_NAME("allocating non-gate selectors"); - vinfo("allocating non-gate selectors"); - - // Set the other non-gate selector polynomials to full size - for (auto& selector : proving_key.polynomials.get_non_gate_selectors()) { - selector = Polynomial(proving_key.circuit_size); - } - } - if constexpr (IsGoblinFlavor) { - PROFILE_THIS_NAME("allocating ecc op wires and selector"); - vinfo("allocating ecc op wires and selector"); - - // Allocate the ecc op wires and selector - const size_t ecc_op_block_size = circuit.blocks.ecc_op.get_fixed_size(is_structured); - const size_t op_wire_offset = Flavor::has_zero_row ? 1 : 0; - for (auto& wire : proving_key.polynomials.get_ecc_op_wires()) { - wire = Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); - } - proving_key.polynomials.lagrange_ecc_op = - Polynomial(ecc_op_block_size, proving_key.circuit_size, op_wire_offset); - } - - if constexpr (HasDataBus) { - // WORKTODO: could be this: - // for (auto& poly : proving_key.polynomials.get_databus_entities()) { - // poly = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - // } - proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.calldata_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata_read_counts = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.secondary_calldata_read_tags = - Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - proving_key.polynomials.return_data_read_tags = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1107): Restricting databus_id to - // databus_size leads to failure. - // const size_t databus_size = std::max({ calldata.size(), secondary_calldata.size(), - // return_data.size() }); - proving_key.polynomials.databus_id = Polynomial(proving_key.circuit_size, proving_key.circuit_size); - } - const size_t max_tables_size = std::min(static_cast(MAX_LOOKUP_TABLES_SIZE), dyadic_circuit_size - 1); - size_t table_offset = dyadic_circuit_size - max_tables_size; - { - PROFILE_THIS_NAME("allocating table polynomials"); - vinfo("allocating table polynomials"); - - ASSERT(dyadic_circuit_size > max_tables_size); - - // Allocate the table polynomials - if constexpr (IsHonkFlavor) { - for (auto& poly : proving_key.polynomials.get_tables()) { - poly = typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - } - } - } - { - PROFILE_THIS_NAME("allocating sigmas and ids"); - vinfo("allocating sigmas and ids"); - - for (auto& sigma : proving_key.polynomials.get_sigmas()) { - sigma = typename Flavor::Polynomial(proving_key.circuit_size); - } - for (auto& id : proving_key.polynomials.get_ids()) { - id = typename Flavor::Polynomial(proving_key.circuit_size); - } - } - { - ZoneScopedN("allocating lookup read counts and tags"); - // Allocate the read counts and tags polynomials - vinfo("allocating lookup read counts and tags"); - proving_key.polynomials.lookup_read_counts = - typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - proving_key.polynomials.lookup_read_tags = - typename Flavor::Polynomial(max_tables_size, dyadic_circuit_size, table_offset); - } - { - ZoneScopedN("allocating lookup and databus inverses"); - // Allocate the lookup_inverses polynomial - vinfo("allocating lookup and databus inverses"); - const size_t lookup_offset = static_cast(circuit.blocks.lookup.trace_offset); - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts - // at top of trace - const size_t table_offset = - dyadic_circuit_size - std::min(dyadic_circuit_size - 1, static_cast(MAX_LOOKUP_TABLES_SIZE)); - const size_t lookup_inverses_start = std::min(lookup_offset, table_offset); - const size_t lookup_inverses_end = - std::min(dyadic_circuit_size, - std::max(lookup_offset + circuit.blocks.lookup.get_fixed_size(is_structured), - table_offset + MAX_LOOKUP_TABLES_SIZE)); - proving_key.polynomials.lookup_inverses = - Polynomial(lookup_inverses_end - lookup_inverses_start, dyadic_circuit_size, lookup_inverses_start); - if constexpr (HasDataBus) { - const size_t q_busread_end = - circuit.blocks.busread.trace_offset + circuit.blocks.busread.get_fixed_size(is_structured); - // Allocate the databus inverse polynomials - proving_key.polynomials.calldata_inverses = - Polynomial(std::max(circuit.get_calldata().size(), q_busread_end), dyadic_circuit_size); - proving_key.polynomials.secondary_calldata_inverses = - Polynomial(std::max(circuit.get_secondary_calldata().size(), q_busread_end), dyadic_circuit_size); - proving_key.polynomials.return_data_inverses = - Polynomial(std::max(circuit.get_return_data().size(), q_busread_end), dyadic_circuit_size); - } - } - { - PROFILE_THIS_NAME("constructing z_perm"); - vinfo("constructing z_perm"); - - // Allocate the z_perm polynomial - proving_key.polynomials.z_perm = Polynomial::shiftable(proving_key.circuit_size); - } - - { - PROFILE_THIS_NAME("allocating lagrange polynomials"); - vinfo("allocating lagrange polynomials"); - - // First and last lagrange polynomials (in the full circuit size) - proving_key.polynomials.lagrange_first = Polynomial(1, dyadic_circuit_size, 0); - proving_key.polynomials.lagrange_last = Polynomial(1, dyadic_circuit_size, dyadic_circuit_size - 1); - } - } }; } // namespace bb From 172aef00aaf0509b4061764ef3e643f0450a8d94 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 5 Nov 2024 19:36:19 +0000 Subject: [PATCH 03/25] basic tests for aux passing w caveats --- .../execution_trace/execution_trace.cpp | 9 +++ .../arithmetization/mega_arithmetization.hpp | 15 +++- .../arithmetization/ultra_arithmetization.hpp | 7 +- .../stdlib_circuit_builders/mock_circuits.hpp | 35 +++++++++ .../barretenberg/ultra_honk/CMakeLists.txt | 2 +- .../ultra_honk/decider_proving_key.hpp | 77 ++++++++++++++++++- .../ultra_honk/mega_composer.test.cpp | 38 ++++++++- 7 files changed, 175 insertions(+), 8 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index 15bbe804221..fb3ba32d91f 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -76,6 +76,15 @@ void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace for (auto& index : builder.memory_write_records) { proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset); } + // info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); + // info("READ records: "); + // for (auto idx : proving_key.memory_read_records) { + // info("idx = ", idx); + // } + // info("WRITE records: "); + // for (auto idx : proving_key.memory_write_records) { + // info("idx = ", idx); + // } } template diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index e05f4f09301..6f1058ca871 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -91,10 +91,10 @@ template class MegaArith { this->ecc_op = 18; this->pub_inputs = 1; this->busread = 3; - this->arithmetic = 7; // 8 - this->delta_range = 2; + this->arithmetic = 15; // 15 + this->delta_range = 5; this->elliptic = 2; - this->aux = 2; + this->aux = 14; // 20 this->poseidon2_external = 2; this->poseidon2_internal = 2; this->lookup = 2; @@ -342,6 +342,15 @@ template class MegaArith { selector.resize(fixed_block_size); } + // WORKTODO: handle mem record offsets + // // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to + // // account for moving them into the miscellaneous block + // if (block.has_ram_rom) { + // uint32_t offset = this->miscellaneous.offset - block.offset + this->miscellaneous.size(); + // for (auto& idx : ) + + // } + this->miscellaneous.has_ram_rom = block.has_ram_rom; this->miscellaneous.is_pub_inputs = block.is_pub_inputs; this->summarize(); diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index 349b41bccec..c445e3f742a 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -23,11 +23,12 @@ template class UltraArith { T lookup; T poseidon2_external; T poseidon2_internal; + T miscellaneous; auto get() { - return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, - aux, lookup, poseidon2_external, poseidon2_internal }; + return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, aux, + lookup, poseidon2_external, poseidon2_internal, miscellaneous }; } auto get_gate_blocks() @@ -51,6 +52,7 @@ template class UltraArith { this->lookup = FIXED_SIZE; this->poseidon2_external = FIXED_SIZE; this->poseidon2_internal = FIXED_SIZE; + this->miscellaneous = 0; } }; @@ -150,6 +152,7 @@ template class UltraArith { info("lookups :\t", this->lookup.size()); info("poseidon ext :\t", this->poseidon2_external.size()); info("poseidon int :\t", this->poseidon2_internal.size()); + info("miscellaneous :\t", this->miscellaneous.size()); } size_t get_total_structured_size() diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp index 0682a8a44ab..0a683a187f0 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp @@ -87,6 +87,41 @@ class MockCircuits { } } + /** + * @brief Add some simple RAM (aux) gates for testing memory read/write functionality + * @brief Each iteration adds + * + * @param builder + */ + template static void add_RAM_gates(Builder& builder) + { + std::array ram_values{ builder.add_variable(5), + builder.add_variable(10), + builder.add_variable(20) }; + + size_t ram_id = builder.create_RAM_array(3); + + for (size_t i = 0; i < 3; ++i) { + builder.init_RAM_element(ram_id, i, ram_values[i]); + } + + auto val_idx_1 = builder.read_RAM_array(ram_id, builder.add_variable(1)); + auto val_idx_2 = builder.read_RAM_array(ram_id, builder.add_variable(2)); + auto val_idx_3 = builder.read_RAM_array(ram_id, builder.add_variable(0)); + + builder.create_big_add_gate({ + val_idx_1, + val_idx_2, + val_idx_3, + builder.zero_idx, + 1, + 1, + 1, + 0, + -35, + }); + } + /** * @brief Populate a builder with a specified number of arithmetic gates; includes a PI * diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt index 7c55050a138..d1fdb5bc8e0 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(ultra_honk sumcheck) \ No newline at end of file +barretenberg_module(ultra_honk sumcheck circuit_checker) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 045e5c3ae9b..00a2bf8363b 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -62,7 +62,8 @@ template class DeciderProvingKey_ { if (is_structured) { circuit.blocks.set_fixed_block_sizes(trace_structure); // set the fixed sizes for each block circuit.blocks.summarize(); - circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size + // circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size + move_structured_trace_overflow_to_miscellaneous_block(circuit); circuit.blocks.summarize(); dyadic_circuit_size = compute_structured_dyadic_size(circuit); // set the dyadic size accordingly } else { @@ -338,6 +339,80 @@ template class DeciderProvingKey_ { void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; + + public: + static void move_structured_trace_overflow_to_miscellaneous_block(Circuit& circuit) + { + using SelectorType = SlabVector; + using WireType = SlabVector; + + auto& blocks = circuit.blocks; + auto& misc_block = circuit.blocks.miscellaneous; + + blocks.compute_offsets(/*is_structured=*/true); + + for (auto& block : blocks.get()) { + size_t block_size = block.size(); + size_t fixed_block_size = block.get_fixed_size(); + if (block_size > fixed_block_size && block != misc_block) { + // We dont handle this case + ASSERT(!block.is_pub_inputs); + + // Set flag indicating that at least one block exceeds capacity and the misc block is in use + blocks.has_overflow = true; + + // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to + // account for moving them into the miscellaneous block + if (block.has_ram_rom) { + + for (auto& val : block.q_aux()) { + info("q_aux = ", val); + } + uint32_t offset = + misc_block.trace_offset - block.trace_offset + static_cast(misc_block.size()); + for (auto& idx : circuit.memory_read_records) { + if (idx >= fixed_block_size) { + idx += offset; + } + } + for (auto& idx : circuit.memory_write_records) { + if (idx >= fixed_block_size) { + idx += offset; + } + } + } + + // move the excess wire and selector data from the offending block to the miscellaneous block + for (auto [wire, misc_wire] : zip_view(block.wires, misc_block.wires)) { + WireType overflow(wire.begin() + static_cast(fixed_block_size), wire.end()); + // misc_wire.push_back(overflow); + for (const auto& val : overflow) { + misc_wire.emplace_back(val); + } + wire.resize(fixed_block_size); + } + for (auto [selector, misc_selector] : zip_view(block.selectors, misc_block.selectors)) { + SelectorType overflow(selector.begin() + static_cast(fixed_block_size), + selector.end()); + // misc_selector.push_back(std::move(overflow)); + for (const auto& val : overflow) { + misc_selector.emplace_back(val); + } + selector.resize(fixed_block_size); + } + + blocks.summarize(); + info(block.size()); + + info("WARNING: Num gates in circuit block exceeds the specified fixed size - execution trace will " + "not be constructed correctly!"); + blocks.summarize(); + // ASSERT(false); + } + } + // Set the fixed size of the miscellaneous block to its current size + misc_block.set_fixed_size(static_cast(misc_block.size())); + } }; } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index b3ed22c903a..dd159646a40 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -2,6 +2,7 @@ #include #include +#include "barretenberg/circuit_checker/circuit_checker.hpp" #include "barretenberg/common/log.hpp" #include "barretenberg/goblin/mock_circuits.hpp" #include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" @@ -214,11 +215,12 @@ TEST_F(MegaHonkTests, MultipleCircuitsHonkAndMerge) * @brief Test proof construction/verification for a structured execution trace * */ -TEST_F(MegaHonkTests, MiscellaneousBlock) +TEST_F(MegaHonkTests, MiscellaneousBlockSimple) { MegaCircuitBuilder builder; GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_arithmetic_gates(builder, 8); // Construct and verify Honk proof using a structured trace TraceStructure trace_structure = TraceStructure::TINY_TEST; @@ -231,3 +233,37 @@ TEST_F(MegaHonkTests, MiscellaneousBlock) auto proof = prover.construct_proof(); EXPECT_TRUE(verifier.verify_proof(proof)); } + +/** + * @brief Test proof construction/verification for a structured execution trace + * + */ +TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) +{ + MegaCircuitBuilder builder; + // using DeciderProvingKey = DeciderProvingKey_; + + TraceStructure trace_structure = TraceStructure::TINY_TEST; + + GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_RAM_gates(builder); + + // builder.blocks.set_fixed_block_sizes(trace_structure); + + // builder.blocks.summarize(); + // DeciderProvingKey::move_structured_trace_overflow_to_miscellaneous_block(builder); + // builder.blocks.summarize(); + + // Construct and verify Honk proof using a structured trace + auto proving_key = std::make_shared>(builder, trace_structure); + + EXPECT_TRUE(CircuitChecker::check(builder)); + + // builder.blocks.summarize(); + MegaProver prover(proving_key); + // builder.blocks.summarize(); + auto verification_key = std::make_shared(proving_key->proving_key); + MegaVerifier verifier(verification_key); + auto proof = prover.construct_proof(); + EXPECT_TRUE(verifier.verify_proof(proof)); +} From 8cfbb6331d175ee9df18707be1457b381d690da4 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 5 Nov 2024 19:59:06 +0000 Subject: [PATCH 04/25] add missing miscellaneous objects --- .../arithmetization/execution_trace_usage_tracker.hpp | 2 +- .../plonk_honk_shared/arithmetization/mega_arithmetization.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp index b1fcfcb0702..7380c7a6e73 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp @@ -93,7 +93,7 @@ struct ExecutionTraceUsageTracker { std::vector block_labels{ "ecc_op", "pub_inputs", "busread", "arithmetic", "delta_range", "elliptic", "aux", "poseidon2_external", "poseidon2_internal", - "lookup" }; + "lookup", "miscellaneous" }; void print() { diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index 6f1058ca871..debaae6f16e 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -48,7 +48,7 @@ template class MegaArith { return RefArray{ ecc_op, pub_inputs, busread, arithmetic, delta_range, elliptic, aux, poseidon2_external, poseidon2_internal, - lookup }; + lookup, miscellaneous }; } auto get_gate_blocks() From dc2a62704065339a4fac5d3c1fe2dfb27e839770 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 5 Nov 2024 22:59:16 +0000 Subject: [PATCH 05/25] correctly updating mem records --- .../execution_trace/execution_trace.cpp | 18 ++++++------- .../arithmetization/mega_arithmetization.hpp | 2 +- .../ultra_honk/decider_proving_key.hpp | 27 +++++++------------ .../ultra_honk/mega_composer.test.cpp | 9 ------- .../barretenberg/ultra_honk/oink_prover.cpp | 17 ++++++++++++ 5 files changed, 37 insertions(+), 36 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index fb3ba32d91f..3a04c48fdc6 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -76,15 +76,15 @@ void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace for (auto& index : builder.memory_write_records) { proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset); } - // info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); - // info("READ records: "); - // for (auto idx : proving_key.memory_read_records) { - // info("idx = ", idx); - // } - // info("WRITE records: "); - // for (auto idx : proving_key.memory_write_records) { - // info("idx = ", idx); - // } + info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); + info("READ records: "); + for (auto idx : proving_key.memory_read_records) { + info("idx = ", idx); + } + info("WRITE records: "); + for (auto idx : proving_key.memory_write_records) { + info("idx = ", idx); + } } template diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index debaae6f16e..baaaca8d678 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -94,7 +94,7 @@ template class MegaArith { this->arithmetic = 15; // 15 this->delta_range = 5; this->elliptic = 2; - this->aux = 14; // 20 + this->aux = 8; // 20 this->poseidon2_external = 2; this->poseidon2_internal = 2; this->lookup = 2; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 00a2bf8363b..00222bad5d0 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -343,9 +343,6 @@ template class DeciderProvingKey_ { public: static void move_structured_trace_overflow_to_miscellaneous_block(Circuit& circuit) { - using SelectorType = SlabVector; - using WireType = SlabVector; - auto& blocks = circuit.blocks; auto& misc_block = circuit.blocks.miscellaneous; @@ -365,11 +362,12 @@ template class DeciderProvingKey_ { // account for moving them into the miscellaneous block if (block.has_ram_rom) { - for (auto& val : block.q_aux()) { - info("q_aux = ", val); - } - uint32_t offset = - misc_block.trace_offset - block.trace_offset + static_cast(misc_block.size()); + // for (auto& val : block.q_aux()) { + // info("q_aux = ", val); + // } + uint32_t offset = misc_block.trace_offset + static_cast(misc_block.size()); + offset -= fixed_block_size; // index was index into aux block + offset -= block.trace_offset; // we'll add block.trace_offset to everything later for (auto& idx : circuit.memory_read_records) { if (idx >= fixed_block_size) { idx += offset; @@ -384,19 +382,14 @@ template class DeciderProvingKey_ { // move the excess wire and selector data from the offending block to the miscellaneous block for (auto [wire, misc_wire] : zip_view(block.wires, misc_block.wires)) { - WireType overflow(wire.begin() + static_cast(fixed_block_size), wire.end()); - // misc_wire.push_back(overflow); - for (const auto& val : overflow) { - misc_wire.emplace_back(val); + for (size_t i = fixed_block_size; i < block_size; ++i) { + misc_wire.emplace_back(wire[i]); } wire.resize(fixed_block_size); } for (auto [selector, misc_selector] : zip_view(block.selectors, misc_block.selectors)) { - SelectorType overflow(selector.begin() + static_cast(fixed_block_size), - selector.end()); - // misc_selector.push_back(std::move(overflow)); - for (const auto& val : overflow) { - misc_selector.emplace_back(val); + for (size_t i = fixed_block_size; i < block_size; ++i) { + misc_selector.emplace_back(selector[i]); } selector.resize(fixed_block_size); } diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index dd159646a40..77234407ee9 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -241,24 +241,15 @@ TEST_F(MegaHonkTests, MiscellaneousBlockSimple) TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) { MegaCircuitBuilder builder; - // using DeciderProvingKey = DeciderProvingKey_; TraceStructure trace_structure = TraceStructure::TINY_TEST; GoblinMockCircuits::construct_simple_circuit(builder); MockCircuits::add_RAM_gates(builder); - // builder.blocks.set_fixed_block_sizes(trace_structure); - - // builder.blocks.summarize(); - // DeciderProvingKey::move_structured_trace_overflow_to_miscellaneous_block(builder); - // builder.blocks.summarize(); - // Construct and verify Honk proof using a structured trace auto proving_key = std::make_shared>(builder, trace_structure); - EXPECT_TRUE(CircuitChecker::check(builder)); - // builder.blocks.summarize(); MegaProver prover(proving_key); // builder.blocks.summarize(); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index b9279d2a5ae..50eccb2c5b8 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -160,8 +160,25 @@ template void OinkProver::execute_sorted_list_acc proving_key->relation_parameters.eta_two = eta_two; proving_key->relation_parameters.eta_three = eta_three; + // auto& polys = proving_key->proving_key.polynomials; + // // size_t idx = 53; // with aux size >8 + // size_t idx = 59; // with aux size >8 + // info("q_aux = ", polys.q_aux[idx]); + // info("q_arith = ", polys.q_arith[idx]); + // info("w_l = ", polys.w_l[idx]); + // info("w_r = ", polys.w_r[idx]); + // info("w_o = ", polys.w_o[idx]); + // info("w_4 = ", polys.w_4[idx]); + proving_key->proving_key.add_ram_rom_memory_records_to_wire_4(eta, eta_two, eta_three); + // info("q_aux = ", polys.q_aux[idx]); + // info("q_arith = ", polys.q_arith[idx]); + // info("w_l = ", polys.w_l[idx]); + // info("w_r = ", polys.w_r[idx]); + // info("w_o = ", polys.w_o[idx]); + // info("w_4 = ", polys.w_4[idx]); + // Commit to lookup argument polynomials and the finalized (i.e. with memory records) fourth wire polynomial { PROFILE_THIS_NAME("COMMIT::lookup_counts_tags"); From 8b7e95506ce212b9e674c65d4d84aec5d9f76304 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Wed, 6 Nov 2024 13:36:23 +0000 Subject: [PATCH 06/25] fix ultra arith --- .../arithmetization/ultra_arithmetization.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index c445e3f742a..4cb78b3cd91 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -137,8 +137,9 @@ template class UltraArith { auto get() { - return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range, this->elliptic, - this->aux, this->lookup, this->poseidon2_external, this->poseidon2_internal }; + return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range, this->elliptic, + this->aux, this->lookup, this->poseidon2_external, this->poseidon2_internal, + this->miscellaneous }; } void summarize() const From b73c7fb1e21141615c441c2279f7fcf3397ce850 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Wed, 6 Nov 2024 18:58:46 +0000 Subject: [PATCH 07/25] fix missing selector ref; aux tests passes at arbitrary split point --- .../arithmetization/mega_arithmetization.hpp | 17 +++++- .../arithmetization/ultra_arithmetization.hpp | 10 ++++ .../ultra_honk/decider_proving_key.hpp | 57 +++++++++++++------ .../barretenberg/ultra_honk/oink_prover.cpp | 33 ++++++----- 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index baaaca8d678..a451dd9ca6d 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -1,5 +1,6 @@ #pragma once +#include "barretenberg/common/ref_vector.hpp" #include "barretenberg/common/zip_view.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp" @@ -94,7 +95,7 @@ template class MegaArith { this->arithmetic = 15; // 15 this->delta_range = 5; this->elliptic = 2; - this->aux = 8; // 20 + this->aux = 10; // 20 this->poseidon2_external = 2; this->poseidon2_internal = 2; this->lookup = 2; @@ -190,6 +191,8 @@ template class MegaArith { using FF = FF_; class MegaTraceBlock : public ExecutionTraceBlock { + using SelectorType = ExecutionTraceBlock::SelectorType; + public: void populate_wires(const uint32_t& idx_1, const uint32_t& idx_2, const uint32_t& idx_3, const uint32_t& idx_4) { @@ -223,6 +226,18 @@ template class MegaArith { auto& q_poseidon2_internal() { return this->selectors[12]; }; auto& q_lookup_type() { return this->selectors[13]; }; + RefVector get_gate_selectors() + { + return { q_busread(), + q_arith(), + q_delta_range(), + q_elliptic(), + q_aux(), + q_poseidon2_external(), + q_poseidon2_internal(), + q_lookup_type() }; + } + /** * @brief Add zeros to all selectors which are not part of the conventional Ultra arithmetization * @details Facilitates reuse of Ultra gate construction functions in arithmetizations which extend the diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index 4cb78b3cd91..b56e8b5381c 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -1,5 +1,6 @@ #pragma once +#include "barretenberg/common/ref_vector.hpp" #include "barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp" namespace bb { @@ -62,6 +63,8 @@ template class UltraArith { using FF = FF_; class UltraTraceBlock : public ExecutionTraceBlock { + using SelectorType = ExecutionTraceBlock::SelectorType; + public: void populate_wires(const uint32_t& idx_1, const uint32_t& idx_2, const uint32_t& idx_3, const uint32_t& idx_4) { @@ -93,6 +96,13 @@ template class UltraArith { auto& q_lookup_type() { return this->selectors[10]; }; auto& q_poseidon2_external() { return this->selectors[11]; }; auto& q_poseidon2_internal() { return this->selectors[12]; }; + + RefVector get_gate_selectors() + { + return { q_arith(), q_delta_range(), q_elliptic(), + q_aux(), q_poseidon2_external(), q_poseidon2_internal(), + q_lookup_type() }; + } }; struct TraceBlocks : public UltraTraceBlocks { diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 00222bad5d0..30552142d36 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -365,46 +365,71 @@ template class DeciderProvingKey_ { // for (auto& val : block.q_aux()) { // info("q_aux = ", val); // } - uint32_t offset = misc_block.trace_offset + static_cast(misc_block.size()); - offset -= fixed_block_size; // index was index into aux block - offset -= block.trace_offset; // we'll add block.trace_offset to everything later + uint32_t misc_block_start_idx = misc_block.trace_offset + static_cast(misc_block.size()); + misc_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later + uint32_t offset = misc_block_start_idx + 1; // +1 accounts for duplication of final gate + bool last_gate_is_memory_read = false; + bool last_gate_is_memory_write = false; for (auto& idx : circuit.memory_read_records) { + if (idx == fixed_block_size - 1) { + last_gate_is_memory_read = true; + info("last_gate_is_memory_read."); + } if (idx >= fixed_block_size) { - idx += offset; + idx -= fixed_block_size; // redefine index from zero + idx += offset; // shift to correct location in misc block } } for (auto& idx : circuit.memory_write_records) { + if (idx == fixed_block_size - 1) { + last_gate_is_memory_write = true; + info("last_gate_is_memory_write."); + } if (idx >= fixed_block_size) { - idx += offset; + idx -= fixed_block_size; // redefine index from zero + idx += offset; // shift to correct location in misc block } } + // The last gate in the main block will be duplicated in the miscellaneous block. If it was a memory + // read/write, add the corresponding the misc block index to the memory record indices. + if (last_gate_is_memory_read) { + circuit.memory_read_records.push_back(misc_block_start_idx); + } else if (last_gate_is_memory_write) { + circuit.memory_write_records.push_back(misc_block_start_idx); + } } - // move the excess wire and selector data from the offending block to the miscellaneous block + // Move the excess wire and selector data from the offending block to the miscellaneous block + size_t start = fixed_block_size - 1; // the final gate in the main block is duplicated + size_t end = block_size; for (auto [wire, misc_wire] : zip_view(block.wires, misc_block.wires)) { - for (size_t i = fixed_block_size; i < block_size; ++i) { + for (size_t i = start; i < end; ++i) { misc_wire.emplace_back(wire[i]); } wire.resize(fixed_block_size); } for (auto [selector, misc_selector] : zip_view(block.selectors, misc_block.selectors)) { - for (size_t i = fixed_block_size; i < block_size; ++i) { + for (size_t i = start; i < end; ++i) { misc_selector.emplace_back(selector[i]); } selector.resize(fixed_block_size); } - - blocks.summarize(); - info(block.size()); - - info("WARNING: Num gates in circuit block exceeds the specified fixed size - execution trace will " - "not be constructed correctly!"); - blocks.summarize(); - // ASSERT(false); + // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors + for (auto& selector : block.get_gate_selectors()) { + selector.back() = 0; + } } } // Set the fixed size of the miscellaneous block to its current size misc_block.set_fixed_size(static_cast(misc_block.size())); + + // WORKTODO: make this spit out a clear display of the original block sizes vs fixed sizes + if (blocks.has_overflow) { + info("WARNING: At least one gate type exceeds the limit defined in the structured trace. The overflow " + "block is in use and may negatively impact performance of the prover!"); + blocks.summarize(); + // ASSERT(false); + } } }; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index 50eccb2c5b8..830b0a5f264 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -161,23 +161,28 @@ template void OinkProver::execute_sorted_list_acc proving_key->relation_parameters.eta_three = eta_three; // auto& polys = proving_key->proving_key.polynomials; - // // size_t idx = 53; // with aux size >8 - // size_t idx = 59; // with aux size >8 - // info("q_aux = ", polys.q_aux[idx]); - // info("q_arith = ", polys.q_arith[idx]); - // info("w_l = ", polys.w_l[idx]); - // info("w_r = ", polys.w_r[idx]); - // info("w_o = ", polys.w_o[idx]); - // info("w_4 = ", polys.w_4[idx]); + // std::vector idxs = { 54, 61 }; + // for (auto idx : idxs) { + // info("IDX = ", idx); + // info("q_aux = ", polys.q_aux[idx]); + // info("q_arith = ", polys.q_arith[idx]); + // info("w_l = ", polys.w_l[idx]); + // info("w_r = ", polys.w_r[idx]); + // info("w_o = ", polys.w_o[idx]); + // info("w_4 = ", polys.w_4[idx]); + // } proving_key->proving_key.add_ram_rom_memory_records_to_wire_4(eta, eta_two, eta_three); - // info("q_aux = ", polys.q_aux[idx]); - // info("q_arith = ", polys.q_arith[idx]); - // info("w_l = ", polys.w_l[idx]); - // info("w_r = ", polys.w_r[idx]); - // info("w_o = ", polys.w_o[idx]); - // info("w_4 = ", polys.w_4[idx]); + // for (auto idx : idxs) { + // info("IDX = ", idx); + // info("q_aux = ", polys.q_aux[idx]); + // info("q_arith = ", polys.q_arith[idx]); + // info("w_l = ", polys.w_l[idx]); + // info("w_r = ", polys.w_r[idx]); + // info("w_o = ", polys.w_o[idx]); + // info("w_4 = ", polys.w_4[idx]); + // } // Commit to lookup argument polynomials and the finalized (i.e. with memory records) fourth wire polynomial { From edfe90ce08f611f6b15c3fbf3558f564a4f79059 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Wed, 6 Nov 2024 23:42:13 +0000 Subject: [PATCH 08/25] WiP introduce trace setting concept --- barretenberg/cpp/src/barretenberg/bb/main.cpp | 6 +- .../client_ivc_bench/client_ivc.bench.cpp | 2 +- .../barretenberg/client_ivc/client_ivc.cpp | 15 ++-- .../barretenberg/client_ivc/client_ivc.hpp | 4 +- .../client_ivc/client_ivc.test.cpp | 69 ++++++++++--------- .../client_ivc_auto_verify.test.cpp | 4 +- .../client_ivc_integration.test.cpp | 10 +-- .../client_ivc/mock_circuit_producer.hpp | 2 +- .../client_ivc/mock_kernel_pinning.test.cpp | 2 +- .../dsl/acir_format/acir_integration.test.cpp | 2 +- .../ivc_recursion_constraint.test.cpp | 8 +-- .../barretenberg/dsl/acir_proofs/c_bind.cpp | 2 +- .../execution_trace/execution_trace.cpp | 18 ++--- .../arithmetization/arithmetization.hpp | 5 ++ .../ultra_honk/decider_proving_key.hpp | 18 +++-- .../ultra_honk/mega_composer.test.cpp | 1 + 16 files changed, 94 insertions(+), 74 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index 3cfad6d4e46..490d8c193df 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -359,7 +359,7 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath, // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::E2E_FULL_TEST; + ivc.trace_settings.structure = TraceStructure::E2E_FULL_TEST; // Accumulate the entire program stack into the IVC // TODO(https://github.com/AztecProtocol/barretenberg/issues/1116): remove manual setting of is_kernel once databus @@ -453,7 +453,7 @@ bool foldAndVerifyProgram(const std::string& bytecodePath, const std::string& wi ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; auto program_stack = acir_format::get_acir_program_stack( bytecodePath, witnessPath, false); // TODO(https://github.com/AztecProtocol/barretenberg/issues/1013): this @@ -507,7 +507,7 @@ void client_ivc_prove_output_all(const std::string& bytecodePath, // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::E2E_FULL_TEST; + ivc.trace_settings.structure = TraceStructure::E2E_FULL_TEST; auto program_stack = acir_format::get_acir_program_stack( bytecodePath, witnessPath, false); // TODO(https://github.com/AztecProtocol/barretenberg/issues/1013): this diff --git a/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp index 12cc8718749..f85bc066ce6 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp @@ -34,7 +34,7 @@ class ClientIVCBench : public benchmark::Fixture { BENCHMARK_DEFINE_F(ClientIVCBench, Full)(benchmark::State& state) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; auto total_num_circuits = 2 * static_cast(state.range(0)); // 2x accounts for kernel circuits auto mocked_vkeys = mock_verification_keys(total_num_circuits); diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp index d5fbf47c5a6..1dae9803b1d 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp @@ -173,11 +173,14 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr proving_key; if (!initialized) { - proving_key = std::make_shared(circuit, trace_structure); - trace_usage_tracker = ExecutionTraceUsageTracker(trace_structure); - } else { proving_key = std::make_shared( - circuit, trace_structure, fold_output.accumulator->proving_key.commitment_key); + circuit, trace_settings.structure, /*commitment_key=*/nullptr, dyadic_circuit_size_override); + trace_usage_tracker = ExecutionTraceUsageTracker(trace_settings.structure); + } else { + proving_key = std::make_shared(circuit, + trace_settings.structure, + fold_output.accumulator->proving_key.commitment_key, + dyadic_circuit_size_override); } // Update the accumulator trace usage based on the present circuit @@ -311,10 +314,10 @@ std::vector> ClientIVC::precompute_f } // Reset the scheme so it can be reused for actual accumulation, maintaining the trace structure setting as is - TraceStructure structure = trace_structure; + TraceSettings settings = trace_settings; bool auto_verify = auto_verify_mode; *this = ClientIVC(); - this->trace_structure = structure; + this->trace_settings = settings; this->auto_verify_mode = auto_verify; return vkeys; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp index 4bad0b49478..0b7b0c9ebc5 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp @@ -106,8 +106,10 @@ class ClientIVC { // Management of linking databus commitments between circuits in the IVC DataBusDepot bus_depot; + size_t dyadic_circuit_size_override = 0; + // A flag indicating whether or not to construct a structured trace in the DeciderProvingKey - TraceStructure trace_structure = TraceStructure::NONE; + TraceSettings trace_settings; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): eventually do away with this. // Setting auto_verify_mode = true will cause kernel completion logic to be added to kernels automatically diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index 370e7015478..cb114dd125a 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -80,7 +80,7 @@ class ClientIVCTests : public ::testing::Test { size_t log2_num_gates = 16) { ClientIVC ivc; // temporary IVC instance needed to produce the complete kernel circuits - ivc.trace_structure = trace_structure; + ivc.trace_settings.structure = trace_structure; std::vector> vkeys; @@ -163,7 +163,7 @@ TEST_F(ClientIVCTests, BadProofFailure) // Confirm that the IVC verifies if nothing is tampered with { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; MockCircuitProducer circuit_producer; @@ -179,7 +179,7 @@ TEST_F(ClientIVCTests, BadProofFailure) // The IVC throws an exception if the FIRST fold proof is tampered with { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; MockCircuitProducer circuit_producer; @@ -204,7 +204,7 @@ TEST_F(ClientIVCTests, BadProofFailure) // The IVC fails if the SECOND fold proof is tampered with { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; MockCircuitProducer circuit_producer; @@ -229,7 +229,7 @@ TEST_F(ClientIVCTests, BadProofFailure) // The IVC fails if the 3rd/FINAL fold proof is tampered with { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; MockCircuitProducer circuit_producer; @@ -278,7 +278,7 @@ TEST_F(ClientIVCTests, BasicLarge) TEST_F(ClientIVCTests, BasicStructured) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; MockCircuitProducer circuit_producer; @@ -325,7 +325,7 @@ TEST_F(ClientIVCTests, PrecomputedVerificationKeys) TEST_F(ClientIVCTests, StructuredPrecomputedVKs) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; size_t NUM_CIRCUITS = 4; size_t log2_num_gates = 5; // number of gates in baseline mocked circuit @@ -333,7 +333,7 @@ TEST_F(ClientIVCTests, StructuredPrecomputedVKs) MockCircuitProducer circuit_producer; auto precomputed_vks = - circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_structure, log2_num_gates); + circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings.structure, log2_num_gates); // Construct and accumulate set of circuits using the precomputed vkeys for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { @@ -357,10 +357,11 @@ TEST(ClientIVCBenchValidation, Full6) bb::srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; size_t total_num_circuits{ 12 }; PrivateFunctionExecutionMockCircuitProducer circuit_producer; - auto precomputed_vkeys = circuit_producer.precompute_verification_keys(total_num_circuits, ivc.trace_structure); + auto precomputed_vkeys = + circuit_producer.precompute_verification_keys(total_num_circuits, ivc.trace_settings.structure); perform_ivc_accumulation_rounds(total_num_circuits, ivc, precomputed_vkeys); auto proof = ivc.prove(); bool verified = verify_ivc(proof, ivc); @@ -377,7 +378,7 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) bb::srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; size_t total_num_circuits{ 12 }; PrivateFunctionExecutionMockCircuitProducer circuit_producer; auto mocked_vkeys = mock_verification_keys(total_num_circuits); @@ -388,26 +389,26 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) ASSERT_NO_FATAL_FAILURE(run_test()); } -// /** -// * @brief Test use of miscellaneous block -// * -// */ -// TEST_F(ClientIVCTests, MiscellaneousBlock) -// { -// ClientIVC ivc; -// ivc.trace_structure = TraceStructure::TINY_TEST; - -// MockCircuitProducer circuit_producer; - -// size_t NUM_CIRCUITS = 2; - -// // Construct and accumulate some circuits of varying size -// size_t log2_num_gates = 5; -// for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { -// auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates); -// ivc.accumulate(circuit); -// log2_num_gates += 2; -// } - -// EXPECT_TRUE(ivc.prove_and_verify()); -// }; \ No newline at end of file +/** + * @brief Test use of miscellaneous block + * + */ +TEST_F(ClientIVCTests, MiscellaneousBlock) +{ + ClientIVC ivc; + ivc.trace_settings = { TraceStructure::SMALL_TEST, /*overflow=*/1 << 10 }; + + MockCircuitProducer circuit_producer; + + size_t NUM_CIRCUITS = 2; + + // Construct and accumulate some circuits of varying size + size_t log2_num_gates = 5; + for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { + auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates); + ivc.accumulate(circuit); + log2_num_gates += 2; + } + + EXPECT_TRUE(ivc.prove_and_verify()); +}; \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_auto_verify.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_auto_verify.test.cpp index dc76f673d21..5064d316794 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_auto_verify.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_auto_verify.test.cpp @@ -132,7 +132,7 @@ TEST_F(ClientIVCAutoVerifyTests, BasicStructured) { ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // Construct some circuits of varying size Builder circuit_0 = create_mock_circuit(ivc, /*is_kernel=*/false, /*log2_num_gates=*/5); @@ -185,7 +185,7 @@ TEST_F(ClientIVCAutoVerifyTests, StructuredPrecomputedVKs) { ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // Construct a set of arbitrary circuits size_t NUM_CIRCUITS = 4; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp index 5b08a08c810..6f1887287ae 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp @@ -37,7 +37,7 @@ class ClientIVCIntegrationTests : public ::testing::Test { TEST_F(ClientIVCIntegrationTests, BenchmarkCaseSimple) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; MockCircuitProducer circuit_producer; @@ -61,7 +61,7 @@ TEST_F(ClientIVCIntegrationTests, BenchmarkCaseSimple) TEST_F(ClientIVCIntegrationTests, ConsecutiveKernels) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; MockCircuitProducer circuit_producer; @@ -89,7 +89,7 @@ TEST_F(ClientIVCIntegrationTests, ConsecutiveKernels) TEST_F(ClientIVCIntegrationTests, BenchmarkCasePrecomputedVKs) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; size_t NUM_CIRCUITS = 6; @@ -97,7 +97,7 @@ TEST_F(ClientIVCIntegrationTests, BenchmarkCasePrecomputedVKs) std::vector> precomputed_vks; { MockCircuitProducer circuit_producer; - precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_structure); + precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings.structure); } MockCircuitProducer circuit_producer; @@ -123,7 +123,7 @@ TEST_F(ClientIVCIntegrationTests, BenchmarkCasePrecomputedVKs) TEST_F(ClientIVCIntegrationTests, DatabusFailure) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; MockCircuitProducer circuit_producer; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp index 7e2332f5309..8fec8712943 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp @@ -138,7 +138,7 @@ class PrivateFunctionExecutionMockCircuitProducer { auto precompute_verification_keys(const size_t num_circuits, TraceStructure trace_structure) { ClientIVC ivc; // temporary IVC instance needed to produce the complete kernel circuits - ivc.trace_structure = trace_structure; + ivc.trace_settings.structure = trace_structure; std::vector> vkeys; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/mock_kernel_pinning.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/mock_kernel_pinning.test.cpp index 2dc33189cbc..c89c773e814 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/mock_kernel_pinning.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/mock_kernel_pinning.test.cpp @@ -24,7 +24,7 @@ class MockKernelTest : public ::testing::Test { TEST_F(MockKernelTest, PinFoldingKernelSizes) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::CLIENT_IVC_BENCH; + ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; MockCircuitProducer circuit_producer; diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp index 3719a9ec15c..14ab23240c8 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_integration.test.cpp @@ -402,7 +402,7 @@ TEST_P(AcirIntegrationFoldingTest, DISABLED_FoldAndVerifyProgramStack) ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; while (!program_stack.empty()) { auto program = program_stack.back(); diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp index 11339b3de7b..285106061be 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp @@ -160,7 +160,7 @@ class IvcRecursionConstraintTest : public ::testing::Test { TEST_F(IvcRecursionConstraintTest, AccumulateTwo) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // construct a mock app_circuit Builder app_circuit = construct_mock_app_circuit(ivc); @@ -185,7 +185,7 @@ TEST_F(IvcRecursionConstraintTest, AccumulateTwo) TEST_F(IvcRecursionConstraintTest, AccumulateFour) { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // construct a mock app_circuit Builder app_circuit_0 = construct_mock_app_circuit(ivc); @@ -232,7 +232,7 @@ TEST_F(IvcRecursionConstraintTest, AccumulateTwoFailure) VerifierInputs alternative_verification_queue_entry; { ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // construct and accumulate a mock app circuit with a single unique public input Builder app_circuit = construct_mock_app_circuit(ivc); @@ -254,7 +254,7 @@ TEST_F(IvcRecursionConstraintTest, AccumulateTwoFailure) // valid) witnesses during constraint system construction VS recursive verifier construction. ClientIVC ivc; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; // construct and accumulate a mock app circuit with a single unique public input Builder app_circuit = construct_mock_app_circuit(ivc); diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp index 16553a32d4a..8def81aa215 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_proofs/c_bind.cpp @@ -97,7 +97,7 @@ WASM_EXPORT void acir_fold_and_verify_program_stack(uint8_t const* acir_vec, ClientIVC ivc; ivc.auto_verify_mode = true; - ivc.trace_structure = TraceStructure::SMALL_TEST; + ivc.trace_settings.structure = TraceStructure::SMALL_TEST; bool is_kernel = false; while (!program_stack.empty()) { diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index 3a04c48fdc6..fb3ba32d91f 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -76,15 +76,15 @@ void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace for (auto& index : builder.memory_write_records) { proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset); } - info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); - info("READ records: "); - for (auto idx : proving_key.memory_read_records) { - info("idx = ", idx); - } - info("WRITE records: "); - for (auto idx : proving_key.memory_write_records) { - info("idx = ", idx); - } + // info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); + // info("READ records: "); + // for (auto idx : proving_key.memory_read_records) { + // info("idx = ", idx); + // } + // info("WRITE records: "); + // for (auto idx : proving_key.memory_write_records) { + // info("idx = ", idx); + // } } template diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index 33d28c664ed..b2bdff6f1d7 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -32,6 +32,11 @@ struct StackTraces { // it is best to use the smallest possible block sizes to accommodate a given situation. enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_FULL_TEST }; +struct TraceSettings { + TraceStructure structure = TraceStructure::NONE; + size_t overflow = 0; +}; + /** * @brief Basic structure for storing gate data in a builder * diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 30552142d36..d1f9553458d 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -45,10 +45,17 @@ template class DeciderProvingKey_ { // The target sum, which is typically nonzero for a ProtogalaxyProver's accmumulator FF target_sum; + // Allows for explicit setting of the circuit size; used to ensure that all circuits accumulated in an IVC have the + // same circuit size in the event that one or more circuits overflow the structured trace. + // WORKTODO: it might make more sense for this to be the overflow size. + size_t dyadic_circuit_size_override = 0; + DeciderProvingKey_(Circuit& circuit, TraceStructure trace_structure = TraceStructure::NONE, - std::shared_ptr commitment_key = nullptr) + std::shared_ptr commitment_key = nullptr, + size_t circuit_size_override = 0) : is_structured(trace_structure != TraceStructure::NONE) + , dyadic_circuit_size_override(circuit_size_override) { PROFILE_THIS_NAME("DeciderProvingKey(Circuit&)"); vinfo("DeciderProvingKey(Circuit&)"); @@ -64,8 +71,9 @@ template class DeciderProvingKey_ { circuit.blocks.summarize(); // circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size move_structured_trace_overflow_to_miscellaneous_block(circuit); - circuit.blocks.summarize(); - dyadic_circuit_size = compute_structured_dyadic_size(circuit); // set the dyadic size accordingly + dyadic_circuit_size = dyadic_circuit_size_override == 0 + ? compute_structured_dyadic_size(circuit) + : dyadic_circuit_size_override; // set the dyadic size accordingly } else { dyadic_circuit_size = compute_dyadic_size(circuit); // set dyadic size directly from circuit block sizes } @@ -373,7 +381,7 @@ template class DeciderProvingKey_ { for (auto& idx : circuit.memory_read_records) { if (idx == fixed_block_size - 1) { last_gate_is_memory_read = true; - info("last_gate_is_memory_read."); + // info("last_gate_is_memory_read."); } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero @@ -383,7 +391,7 @@ template class DeciderProvingKey_ { for (auto& idx : circuit.memory_write_records) { if (idx == fixed_block_size - 1) { last_gate_is_memory_write = true; - info("last_gate_is_memory_write."); + // info("last_gate_is_memory_write."); } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index 77234407ee9..e6d5930e16b 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -245,6 +245,7 @@ TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) TraceStructure trace_structure = TraceStructure::TINY_TEST; GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_arithmetic_gates(builder, 8); MockCircuits::add_RAM_gates(builder); // Construct and verify Honk proof using a structured trace From 97d4ad672e00be841df88a1395f1a07888d8cfa4 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 03:54:35 +0000 Subject: [PATCH 09/25] everything building with trace settings --- .../barretenberg/client_ivc/client_ivc.cpp | 10 ++++----- .../goblin/mock_circuits_pinning.test.cpp | 9 +++++--- .../arithmetization/arithmetization.hpp | 2 +- .../execution_trace_usage_tracker.hpp | 12 +++++----- .../arithmetization/mega_arithmetization.hpp | 5 +++-- .../arithmetization/ultra_arithmetization.hpp | 5 +++-- .../protogalaxy/protogalaxy.test.cpp | 22 +++++++++---------- .../ultra_honk/decider_proving_key.hpp | 6 ++--- .../ultra_honk/mega_composer.test.cpp | 12 +++++----- .../ultra_honk/ultra_honk.test.cpp | 4 ++-- .../vm/avm/tests/recursive_verifier.test.cpp | 2 +- 11 files changed, 46 insertions(+), 43 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp index 1dae9803b1d..df025567bbe 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp @@ -174,13 +174,11 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr proving_key; if (!initialized) { proving_key = std::make_shared( - circuit, trace_settings.structure, /*commitment_key=*/nullptr, dyadic_circuit_size_override); - trace_usage_tracker = ExecutionTraceUsageTracker(trace_settings.structure); + circuit, trace_settings, /*commitment_key=*/nullptr, dyadic_circuit_size_override); + trace_usage_tracker = ExecutionTraceUsageTracker(trace_settings); } else { - proving_key = std::make_shared(circuit, - trace_settings.structure, - fold_output.accumulator->proving_key.commitment_key, - dyadic_circuit_size_override); + proving_key = std::make_shared( + circuit, trace_settings, fold_output.accumulator->proving_key.commitment_key, dyadic_circuit_size_override); } // Update the accumulator trace usage based on the present circuit diff --git a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp index 53e7c5b2644..afba3ac5402 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp +++ b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits_pinning.test.cpp @@ -58,7 +58,8 @@ TEST_F(MegaMockCircuitsPinning, SmallTestStructuredCircuitSize) { GoblinProver goblin; MegaCircuitBuilder app_circuit{ goblin.op_queue }; - auto proving_key = std::make_shared(app_circuit, TraceStructure::SMALL_TEST); + TraceSettings trace_settings{ TraceStructure::SMALL_TEST }; + auto proving_key = std::make_shared(app_circuit, trace_settings); EXPECT_EQ(proving_key->proving_key.log_circuit_size, 18); } @@ -66,7 +67,8 @@ TEST_F(MegaMockCircuitsPinning, ClientIVCBenchStructuredCircuitSize) { GoblinProver goblin; MegaCircuitBuilder app_circuit{ goblin.op_queue }; - auto proving_key = std::make_shared(app_circuit, TraceStructure::CLIENT_IVC_BENCH); + TraceSettings trace_settings{ TraceStructure::CLIENT_IVC_BENCH }; + auto proving_key = std::make_shared(app_circuit, trace_settings); EXPECT_EQ(proving_key->proving_key.log_circuit_size, 19); } @@ -74,6 +76,7 @@ TEST_F(MegaMockCircuitsPinning, E2EStructuredCircuitSize) { GoblinProver goblin; MegaCircuitBuilder app_circuit{ goblin.op_queue }; - auto proving_key = std::make_shared(app_circuit, TraceStructure::E2E_FULL_TEST); + TraceSettings trace_settings{ TraceStructure::E2E_FULL_TEST }; + auto proving_key = std::make_shared(app_circuit, trace_settings); EXPECT_EQ(proving_key->proving_key.log_circuit_size, 20); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index b2bdff6f1d7..52fdf03a0dc 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -34,7 +34,7 @@ enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_F struct TraceSettings { TraceStructure structure = TraceStructure::NONE; - size_t overflow = 0; + uint32_t overflow = 0; }; /** diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp index 7380c7a6e73..e50298970e5 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp @@ -29,15 +29,15 @@ struct ExecutionTraceUsageTracker { size_t max_databus_size = 0; size_t max_tables_size = 0; - TraceStructure trace_structure = TraceStructure::NONE; + TraceSettings trace_settings; - ExecutionTraceUsageTracker(const TraceStructure& trace_structure = TraceStructure::NONE) - : trace_structure(trace_structure) + ExecutionTraceUsageTracker(const TraceSettings& trace_settings = TraceSettings{}) + : trace_settings(trace_settings) { for (auto& size : max_sizes.get()) { size = 0; // init max sizes to zero } - fixed_sizes.set_fixed_block_sizes(trace_structure); + fixed_sizes.set_fixed_block_sizes(trace_settings); fixed_sizes.compute_offsets(/*is_structured=*/true); } @@ -78,7 +78,7 @@ struct ExecutionTraceUsageTracker { bool check_is_active(const size_t idx) { // If structured trace is not in use, assume the whole trace is active - if (trace_structure == TraceStructure::NONE) { + if (trace_settings.structure == TraceStructure::NONE) { return true; } for (auto& range : active_ranges.get()) { @@ -140,7 +140,7 @@ struct ExecutionTraceUsageTracker { // Convert the active ranges for each gate type into a set of sorted non-overlapping ranges (union of the input) std::vector simplified_active_ranges; - if (trace_structure == TraceStructure::NONE) { + if (trace_settings.structure == TraceStructure::NONE) { // If not using a structured trace, set the active range to the whole domain simplified_active_ranges.push_back(Range{ 0, full_domain_size }); } else { diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index a451dd9ca6d..3ea7b2f8cd8 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -266,11 +266,11 @@ template class MegaArith { } // Set fixed block sizes for use in structured trace - void set_fixed_block_sizes(TraceStructure setting) + void set_fixed_block_sizes(TraceSettings settings) { MegaTraceBlocks fixed_block_sizes{}; // zero initialized - switch (setting) { + switch (settings.structure) { case TraceStructure::NONE: break; case TraceStructure::TINY_TEST: @@ -289,6 +289,7 @@ template class MegaArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } + this->miscellaneous.set_fixed_size(settings.overflow); } void compute_offsets(bool is_structured) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index b56e8b5381c..95d9da67415 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -116,11 +116,11 @@ template class UltraArith { } // Set fixed block sizes for use in structured trace - void set_fixed_block_sizes(TraceStructure setting) + void set_fixed_block_sizes(TraceSettings settings) { UltraTraceBlocks fixed_block_sizes{}; // zero initialized - switch (setting) { + switch (settings.structure) { case TraceStructure::NONE: break; // We don't use Ultra in ClientIvc so no need for anything other than sizing for simple unit tests @@ -134,6 +134,7 @@ template class UltraArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } + this->miscellaneous.set_fixed_size(settings.overflow); } void compute_offsets(bool is_structured) diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp index 0483f2214a8..4db7309a7da 100644 --- a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp +++ b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy.test.cpp @@ -54,10 +54,10 @@ template class ProtogalaxyTests : public testing::Test { } // Construct decider keys for a provided circuit and add to tuple - static void construct_keys(TupleOfKeys& keys, Builder& builder, TraceStructure structure = TraceStructure::NONE) + static void construct_keys(TupleOfKeys& keys, Builder& builder, TraceSettings trace_settings = TraceSettings{}) { - auto decider_proving_key = std::make_shared(builder, structure); + auto decider_proving_key = std::make_shared(builder, trace_settings); auto verification_key = std::make_shared(decider_proving_key->proving_key); auto decider_verification_keys = std::make_shared(verification_key); get<0>(keys).emplace_back(decider_proving_key); @@ -65,7 +65,7 @@ template class ProtogalaxyTests : public testing::Test { } // Construct a given numer of decider key pairs - static TupleOfKeys construct_keys(size_t num_keys, TraceStructure structure = TraceStructure::NONE) + static TupleOfKeys construct_keys(size_t num_keys, TraceSettings trace_settings = TraceSettings{}) { TupleOfKeys keys; // TODO(https://github.com/AztecProtocol/barretenberg/issues/938): Parallelize this loop @@ -73,7 +73,7 @@ template class ProtogalaxyTests : public testing::Test { auto builder = typename Flavor::CircuitBuilder(); construct_circuit(builder); - construct_keys(keys, builder, structure); + construct_keys(keys, builder, trace_settings); } return keys; } @@ -442,13 +442,13 @@ template class ProtogalaxyTests : public testing::Test { */ static void test_full_protogalaxy_structured_trace() { - TraceStructure trace_structure = TraceStructure::SMALL_TEST; - TupleOfKeys keys_1 = construct_keys(2, trace_structure); + TraceSettings trace_settings{ TraceStructure::SMALL_TEST }; + TupleOfKeys keys_1 = construct_keys(2, trace_settings); auto [prover_accumulator, verifier_accumulator] = fold_and_verify(get<0>(keys_1), get<1>(keys_1)); check_accumulator_target_sum_manual(prover_accumulator, true); - TupleOfKeys keys_2 = construct_keys(1, trace_structure); // just one key pair + TupleOfKeys keys_2 = construct_keys(1, trace_settings); // just one key pair auto [prover_accumulator_2, verifier_accumulator_2] = fold_and_verify({ prover_accumulator, get<0>(keys_2)[0] }, { verifier_accumulator, get<1>(keys_2)[0] }); @@ -465,7 +465,7 @@ template class ProtogalaxyTests : public testing::Test { */ static void test_full_protogalaxy_structured_trace_inhomogeneous_circuits() { - TraceStructure trace_structure = TraceStructure::SMALL_TEST; + TraceSettings trace_settings{ TraceStructure::SMALL_TEST }; // Construct three circuits to be folded, each with a different number of constraints Builder builder1; @@ -482,8 +482,8 @@ template class ProtogalaxyTests : public testing::Test { // Construct the decider key pairs for the first two circuits TupleOfKeys keys_1; - construct_keys(keys_1, builder1, trace_structure); - construct_keys(keys_1, builder2, trace_structure); + construct_keys(keys_1, builder1, trace_settings); + construct_keys(keys_1, builder2, trace_settings); // Fold the first two pairs auto [prover_accumulator, verifier_accumulator] = fold_and_verify(get<0>(keys_1), get<1>(keys_1)); @@ -491,7 +491,7 @@ template class ProtogalaxyTests : public testing::Test { // Construct the decider key pair for the third circuit TupleOfKeys keys_2; - construct_keys(keys_2, builder3, trace_structure); + construct_keys(keys_2, builder3, trace_settings); // Fold 3rd pair of keys into their respective accumulators auto [prover_accumulator_2, verifier_accumulator_2] = diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index d1f9553458d..bbcd99fcaa1 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -51,10 +51,10 @@ template class DeciderProvingKey_ { size_t dyadic_circuit_size_override = 0; DeciderProvingKey_(Circuit& circuit, - TraceStructure trace_structure = TraceStructure::NONE, + TraceSettings trace_settings = TraceSettings{}, std::shared_ptr commitment_key = nullptr, size_t circuit_size_override = 0) - : is_structured(trace_structure != TraceStructure::NONE) + : is_structured(trace_settings.structure != TraceStructure::NONE) , dyadic_circuit_size_override(circuit_size_override) { PROFILE_THIS_NAME("DeciderProvingKey(Circuit&)"); @@ -67,7 +67,7 @@ template class DeciderProvingKey_ { // If using a structured trace, set fixed block sizes, check their validity, and set the dyadic circuit size if (is_structured) { - circuit.blocks.set_fixed_block_sizes(trace_structure); // set the fixed sizes for each block + circuit.blocks.set_fixed_block_sizes(trace_settings); // set the fixed sizes for each block circuit.blocks.summarize(); // circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size move_structured_trace_overflow_to_miscellaneous_block(circuit); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index e6d5930e16b..56662766f6e 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -87,8 +87,8 @@ TEST_F(MegaHonkTests, BasicStructured) GoblinMockCircuits::construct_simple_circuit(builder); // Construct and verify Honk proof using a structured trace - TraceStructure trace_structure = TraceStructure::SMALL_TEST; - auto proving_key = std::make_shared>(builder, trace_structure); + TraceSettings trace_settings{ TraceStructure::SMALL_TEST }; + auto proving_key = std::make_shared>(builder, trace_settings); MegaProver prover(proving_key); auto verification_key = std::make_shared(proving_key->proving_key); MegaVerifier verifier(verification_key); @@ -223,8 +223,8 @@ TEST_F(MegaHonkTests, MiscellaneousBlockSimple) MockCircuits::add_arithmetic_gates(builder, 8); // Construct and verify Honk proof using a structured trace - TraceStructure trace_structure = TraceStructure::TINY_TEST; - auto proving_key = std::make_shared>(builder, trace_structure); + TraceSettings trace_settings{ TraceStructure::TINY_TEST }; + auto proving_key = std::make_shared>(builder, trace_settings); // builder.blocks.summarize(); MegaProver prover(proving_key); // builder.blocks.summarize(); @@ -242,14 +242,14 @@ TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) { MegaCircuitBuilder builder; - TraceStructure trace_structure = TraceStructure::TINY_TEST; + TraceSettings trace_settings{ TraceStructure::TINY_TEST }; GoblinMockCircuits::construct_simple_circuit(builder); MockCircuits::add_arithmetic_gates(builder, 8); MockCircuits::add_RAM_gates(builder); // Construct and verify Honk proof using a structured trace - auto proving_key = std::make_shared>(builder, trace_structure); + auto proving_key = std::make_shared>(builder, trace_settings); // builder.blocks.summarize(); MegaProver prover(proving_key); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_honk.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_honk.test.cpp index 79f6406cbe6..b8564e9ae75 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_honk.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_honk.test.cpp @@ -100,8 +100,8 @@ TYPED_TEST(UltraHonkTests, StructuredTrace) MockCircuits::add_arithmetic_gates_with_public_inputs(builder, num_gates); // Construct an proving_key with a structured execution trace - TraceStructure trace_structure = TraceStructure::SMALL_TEST; - auto proving_key = std::make_shared(builder, trace_structure); + TraceSettings trace_settings{ TraceStructure::SMALL_TEST }; + auto proving_key = std::make_shared(builder, trace_settings); typename TestFixture::Prover prover(proving_key); auto verification_key = std::make_shared(proving_key->proving_key); typename TestFixture::Verifier verifier(verification_key); diff --git a/barretenberg/cpp/src/barretenberg/vm/avm/tests/recursive_verifier.test.cpp b/barretenberg/cpp/src/barretenberg/vm/avm/tests/recursive_verifier.test.cpp index 1de397a31f2..d6c110e3499 100644 --- a/barretenberg/cpp/src/barretenberg/vm/avm/tests/recursive_verifier.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm/avm/tests/recursive_verifier.test.cpp @@ -120,7 +120,7 @@ TEST_F(AvmRecursiveTests, recursion) // Make a proof of the verification of an AVM proof const size_t srs_size = 1 << 23; auto ultra_instance = std::make_shared( - outer_circuit, TraceStructure::NONE, std::make_shared>(srs_size)); + outer_circuit, TraceSettings{}, std::make_shared>(srs_size)); OuterProver ultra_prover(ultra_instance); auto ultra_verification_key = std::make_shared(ultra_instance->proving_key); OuterVerifier ultra_verifier(ultra_verification_key); From 354761cc603f9c4aea4de6de7d0482fbd097ff7c Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 15:43:46 +0000 Subject: [PATCH 10/25] basic IVC overflow test passes --- .../client_ivc/client_ivc.test.cpp | 4 ++-- .../arithmetization/mega_arithmetization.hpp | 1 + .../ultra_honk/decider_proving_key.hpp | 20 ++++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index cb114dd125a..3cb04efba78 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -396,14 +396,14 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) TEST_F(ClientIVCTests, MiscellaneousBlock) { ClientIVC ivc; - ivc.trace_settings = { TraceStructure::SMALL_TEST, /*overflow=*/1 << 10 }; + ivc.trace_settings = { TraceStructure::SMALL_TEST, /*overflow=*/1 << 17 }; MockCircuitProducer circuit_producer; size_t NUM_CIRCUITS = 2; // Construct and accumulate some circuits of varying size - size_t log2_num_gates = 5; + size_t log2_num_gates = 14; for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates); ivc.accumulate(circuit); diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index 3ea7b2f8cd8..ce1c3da7a5d 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -289,6 +289,7 @@ template class MegaArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } + // Set the size of miscellaneous block containing the overflow from all other blocks this->miscellaneous.set_fixed_size(settings.overflow); } diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index bbcd99fcaa1..87f3101be7f 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -354,6 +354,13 @@ template class DeciderProvingKey_ { auto& blocks = circuit.blocks; auto& misc_block = circuit.blocks.miscellaneous; + // WORKTODO: need to think about what cases to support here. Can't be surprised by an overflow in the IVC + // setting but need to be able to determine the overflow for a one-off circuit. + if (misc_block.get_fixed_size() > 0) { + info("Miscellaneous block has non-zero size; setting has_overflow to TRUE."); + blocks.has_overflow = true; + } + blocks.compute_offsets(/*is_structured=*/true); for (auto& block : blocks.get()) { @@ -428,8 +435,19 @@ template class DeciderProvingKey_ { } } } + // WORKTODO: probably don't set this here but this means it needs to be set correctly on input // Set the fixed size of the miscellaneous block to its current size - misc_block.set_fixed_size(static_cast(misc_block.size())); + if (misc_block.size() > misc_block.get_fixed_size()) { + info("WARNING: Overflow value was set too low! Miscellaneous block fixed size: ", + misc_block.get_fixed_size(), + ". Miscellaneous block actual size: ", + misc_block.size()); + // Cannot currently support dynamic resizing of the miscellaneous block since this would require dynamic + // expansion of the accumulator in the IVC setting. + // ASSERT(false); + info("Setting miscellaneous block size to: ", misc_block.size()); + misc_block.set_fixed_size(static_cast(misc_block.size())); + } // WORKTODO: make this spit out a clear display of the original block sizes vs fixed sizes if (blocks.has_overflow) { From 3dbcd5a4f762de7354764838c8bddf3f1059da19 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 15:59:47 +0000 Subject: [PATCH 11/25] remove circuit size pverride mechanism --- .../cpp/src/barretenberg/client_ivc/client_ivc.cpp | 5 ++--- .../cpp/src/barretenberg/client_ivc/client_ivc.hpp | 2 -- .../barretenberg/ultra_honk/decider_proving_key.hpp | 13 ++----------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp index df025567bbe..f6179f456b3 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp @@ -173,12 +173,11 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr proving_key; if (!initialized) { - proving_key = std::make_shared( - circuit, trace_settings, /*commitment_key=*/nullptr, dyadic_circuit_size_override); + proving_key = std::make_shared(circuit, trace_settings, /*commitment_key=*/nullptr); trace_usage_tracker = ExecutionTraceUsageTracker(trace_settings); } else { proving_key = std::make_shared( - circuit, trace_settings, fold_output.accumulator->proving_key.commitment_key, dyadic_circuit_size_override); + circuit, trace_settings, fold_output.accumulator->proving_key.commitment_key); } // Update the accumulator trace usage based on the present circuit diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp index 0b7b0c9ebc5..c2c9b14bd23 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp @@ -106,8 +106,6 @@ class ClientIVC { // Management of linking databus commitments between circuits in the IVC DataBusDepot bus_depot; - size_t dyadic_circuit_size_override = 0; - // A flag indicating whether or not to construct a structured trace in the DeciderProvingKey TraceSettings trace_settings; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 87f3101be7f..4405e653eff 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -45,17 +45,10 @@ template class DeciderProvingKey_ { // The target sum, which is typically nonzero for a ProtogalaxyProver's accmumulator FF target_sum; - // Allows for explicit setting of the circuit size; used to ensure that all circuits accumulated in an IVC have the - // same circuit size in the event that one or more circuits overflow the structured trace. - // WORKTODO: it might make more sense for this to be the overflow size. - size_t dyadic_circuit_size_override = 0; - DeciderProvingKey_(Circuit& circuit, TraceSettings trace_settings = TraceSettings{}, - std::shared_ptr commitment_key = nullptr, - size_t circuit_size_override = 0) + std::shared_ptr commitment_key = nullptr) : is_structured(trace_settings.structure != TraceStructure::NONE) - , dyadic_circuit_size_override(circuit_size_override) { PROFILE_THIS_NAME("DeciderProvingKey(Circuit&)"); vinfo("DeciderProvingKey(Circuit&)"); @@ -71,9 +64,7 @@ template class DeciderProvingKey_ { circuit.blocks.summarize(); // circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size move_structured_trace_overflow_to_miscellaneous_block(circuit); - dyadic_circuit_size = dyadic_circuit_size_override == 0 - ? compute_structured_dyadic_size(circuit) - : dyadic_circuit_size_override; // set the dyadic size accordingly + dyadic_circuit_size = compute_structured_dyadic_size(circuit); // set the dyadic size accordingly } else { dyadic_circuit_size = compute_dyadic_size(circuit); // set dyadic size directly from circuit block sizes } From 8b8154928cd4faa4a070d6ca1eb01acd18be3c47 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 17:43:54 +0000 Subject: [PATCH 12/25] naming change to overflow --- .../client_ivc/client_ivc.test.cpp | 2 +- .../execution_trace_usage_tracker.hpp | 15 ++- .../arithmetization/mega_arithmetization.hpp | 110 ++++++------------ .../arithmetization/ultra_arithmetization.hpp | 34 ++---- .../ultra_honk/decider_proving_key.hpp | 60 +++++----- 5 files changed, 84 insertions(+), 137 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index 3cb04efba78..1582f2f91db 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -390,7 +390,7 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) } /** - * @brief Test use of miscellaneous block + * @brief Test use of overflow block * */ TEST_F(ClientIVCTests, MiscellaneousBlock) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp index e50298970e5..53db481ff59 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/execution_trace_usage_tracker.hpp @@ -90,10 +90,17 @@ struct ExecutionTraceUsageTracker { } // For printing only. Must match the order of the members in the arithmetization - std::vector block_labels{ "ecc_op", "pub_inputs", "busread", - "arithmetic", "delta_range", "elliptic", - "aux", "poseidon2_external", "poseidon2_internal", - "lookup", "miscellaneous" }; + std::vector block_labels{ "ecc_op", + "pub_inputs", + "busread", + "arithmetic", + "delta_range", + "elliptic", + "aux", + "poseidon2_external", + "poseidon2_internal", + "lookup", + "overflow" }; void print() { diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index ce1c3da7a5d..a12a01033fd 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -35,21 +35,35 @@ template class MegaArith { T poseidon2_external; T poseidon2_internal; T lookup; - T miscellaneous; + T overflow; auto get() { - return RefArray{ ecc_op, pub_inputs, busread, - arithmetic, delta_range, elliptic, - aux, poseidon2_external, poseidon2_internal, - lookup, miscellaneous }; + return RefArray{ ecc_op, + pub_inputs, + busread, + arithmetic, + delta_range, + elliptic, + aux, + poseidon2_external, + poseidon2_internal, + lookup, + overflow }; } auto get() const { - return RefArray{ ecc_op, pub_inputs, busread, - arithmetic, delta_range, elliptic, - aux, poseidon2_external, poseidon2_internal, - lookup, miscellaneous }; + return RefArray{ ecc_op, + pub_inputs, + busread, + arithmetic, + delta_range, + elliptic, + aux, + poseidon2_external, + poseidon2_internal, + lookup, + overflow }; } auto get_gate_blocks() @@ -76,7 +90,7 @@ template class MegaArith { // this->poseidon2_external = FIXED_SIZE; // this->poseidon2_internal = 1 << 15; // this->lookup = FIXED_SIZE; - // this->miscellaneous = 0; + // this->overflow = 0; // this->ecc_op = 50; // this->pub_inputs = 33; @@ -99,7 +113,7 @@ template class MegaArith { this->poseidon2_external = 2; this->poseidon2_internal = 2; this->lookup = 2; - this->miscellaneous = 0; + this->overflow = 0; } }; // An arbitrary but small-ish structuring that can be used for generic unit testing with non-trivial circuits @@ -117,7 +131,7 @@ template class MegaArith { this->poseidon2_external = FIXED_SIZE; this->poseidon2_internal = 1 << 15; this->lookup = FIXED_SIZE; - this->miscellaneous = 0; + this->overflow = 0; } }; @@ -136,7 +150,7 @@ template class MegaArith { this->poseidon2_external = 2500; this->poseidon2_internal = 14000; this->lookup = 72000; - this->miscellaneous = 0; + this->overflow = 0; // Additional structurings for testing // // 2^18 (Only viable if no 2^19 circuit is used!) @@ -150,7 +164,7 @@ template class MegaArith { // this->poseidon2_external = 2500; // this->poseidon2_internal = 14000; // this->lookup = 36000; - // this->miscellaneous = 0; + // this->overflow = 0; // // 2^20 // this->ecc_op = 1 << 11; @@ -163,7 +177,7 @@ template class MegaArith { // this->poseidon2_external = 5000; // this->poseidon2_internal = 28000; // this->lookup = 144000; - // this->miscellaneous = 0; + // this->overflow = 0; } }; @@ -181,7 +195,7 @@ template class MegaArith { this->poseidon2_external = 30128; this->poseidon2_internal = 172000; this->lookup = 200000; - this->miscellaneous = 0; + this->overflow = 0; } }; @@ -289,8 +303,8 @@ template class MegaArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } - // Set the size of miscellaneous block containing the overflow from all other blocks - this->miscellaneous.set_fixed_size(settings.overflow); + // Set the size of overflow block containing the overflow from all other blocks + this->overflow.set_fixed_size(settings.overflow); } void compute_offsets(bool is_structured) @@ -315,7 +329,7 @@ template class MegaArith { info("poseidon ext :\t", this->poseidon2_external.size(), "/", this->poseidon2_external.get_fixed_size()); info("poseidon int :\t", this->poseidon2_internal.size(), "/", this->poseidon2_internal.get_fixed_size()); info("lookups :\t", this->lookup.size(), "/", this->lookup.get_fixed_size()); - info("miscellaneous :\t", this->miscellaneous.size(), "/", this->miscellaneous.get_fixed_size()); + info("overflow :\t", this->overflow.size(), "/", this->overflow.get_fixed_size()); info(""); } @@ -328,64 +342,6 @@ template class MegaArith { return total_size; } - void check_within_fixed_sizes() - { - using SelectorType = SlabVector; - using WireType = SlabVector; - - for (auto& block : this->get()) { - size_t block_size = block.size(); - size_t fixed_block_size = block.get_fixed_size(); - if (block_size > block.get_fixed_size() && block != this->miscellaneous) { - // Set flag indicating that at least one block exceeds capacity and the misc block is in use - this->has_overflow = true; - - // move the excess wire and selector data from the offending block to the miscellaneous block - for (auto [wire, misc_wire] : zip_view(block.wires, this->miscellaneous.wires)) { - WireType overflow(wire.begin() + static_cast(fixed_block_size), wire.end()); - // misc_wire.push_back(overflow); - for (const auto& val : overflow) { - misc_wire.emplace_back(val); - } - wire.resize(fixed_block_size); - } - for (auto [selector, misc_selector] : zip_view(block.selectors, this->miscellaneous.selectors)) { - SelectorType overflow(selector.begin() + static_cast(fixed_block_size), - selector.end()); - // misc_selector.push_back(std::move(overflow)); - for (const auto& val : overflow) { - misc_selector.emplace_back(val); - } - selector.resize(fixed_block_size); - } - - // WORKTODO: handle mem record offsets - // // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to - // // account for moving them into the miscellaneous block - // if (block.has_ram_rom) { - // uint32_t offset = this->miscellaneous.offset - block.offset + this->miscellaneous.size(); - // for (auto& idx : ) - - // } - - this->miscellaneous.has_ram_rom = block.has_ram_rom; - this->miscellaneous.is_pub_inputs = block.is_pub_inputs; - this->summarize(); - info(block.size()); - - info("WARNING: Num gates in circuit block exceeds the specified fixed size - execution trace will " - "not be constructed correctly!"); - summarize(); - // ASSERT(false); - } - } - // WORKTODO: for now we dont handle this - ASSERT(!this->miscellaneous.has_ram_rom); - ASSERT(!this->miscellaneous.is_pub_inputs); - // Set the fixed size of the miscellaneous block to its current size - this->miscellaneous.set_fixed_size(static_cast(this->miscellaneous.size())); - } - bool operator==(const TraceBlocks& other) const = default; }; diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index 95d9da67415..e2f46c29416 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -24,12 +24,12 @@ template class UltraArith { T lookup; T poseidon2_external; T poseidon2_internal; - T miscellaneous; + T overflow; auto get() { - return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, aux, - lookup, poseidon2_external, poseidon2_internal, miscellaneous }; + return RefArray{ pub_inputs, arithmetic, delta_range, elliptic, aux, + lookup, poseidon2_external, poseidon2_internal, overflow }; } auto get_gate_blocks() @@ -53,7 +53,7 @@ template class UltraArith { this->lookup = FIXED_SIZE; this->poseidon2_external = FIXED_SIZE; this->poseidon2_internal = FIXED_SIZE; - this->miscellaneous = 0; + this->overflow = 0; } }; @@ -134,7 +134,7 @@ template class UltraArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } - this->miscellaneous.set_fixed_size(settings.overflow); + this->overflow.set_fixed_size(settings.overflow); } void compute_offsets(bool is_structured) @@ -148,9 +148,9 @@ template class UltraArith { auto get() { - return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range, this->elliptic, - this->aux, this->lookup, this->poseidon2_external, this->poseidon2_internal, - this->miscellaneous }; + return RefArray{ this->pub_inputs, this->arithmetic, this->delta_range, this->elliptic, + this->aux, this->lookup, this->poseidon2_external, this->poseidon2_internal, + this->overflow }; } void summarize() const @@ -164,7 +164,7 @@ template class UltraArith { info("lookups :\t", this->lookup.size()); info("poseidon ext :\t", this->poseidon2_external.size()); info("poseidon int :\t", this->poseidon2_internal.size()); - info("miscellaneous :\t", this->miscellaneous.size()); + info("overflow :\t", this->overflow.size()); } size_t get_total_structured_size() @@ -176,22 +176,6 @@ template class UltraArith { return total_size; } - /** - * @brief Check that the number of rows populated in each block does not exceed the specified fixed size - * @note This check is only applicable when utilizing a structured trace - * - */ - void check_within_fixed_sizes() - { - for (auto block : this->get()) { - if (block.size() > block.get_fixed_size()) { - info("WARNING: Num gates in circuit block exceeds the specified fixed size - execution trace will " - "not be constructed correctly!"); - ASSERT(false); - } - } - } - bool operator==(const TraceBlocks& other) const = default; }; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 4405e653eff..853975ef0e0 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -62,8 +62,7 @@ template class DeciderProvingKey_ { if (is_structured) { circuit.blocks.set_fixed_block_sizes(trace_settings); // set the fixed sizes for each block circuit.blocks.summarize(); - // circuit.blocks.check_within_fixed_sizes(); // ensure that no block exceeds its fixed size - move_structured_trace_overflow_to_miscellaneous_block(circuit); + move_structured_trace_overflow_to_overflow_block(circuit); dyadic_circuit_size = compute_structured_dyadic_size(circuit); // set the dyadic size accordingly } else { dyadic_circuit_size = compute_dyadic_size(circuit); // set dyadic size directly from circuit block sizes @@ -91,7 +90,7 @@ template class DeciderProvingKey_ { // Allocate full size polynomials proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); } - // If using structured trace but overflow has occurred (misc. block in use), allocate full size polynomials + // If using structured trace but overflow has occurred (overflow block in use), allocate full size polys else if (is_structured && circuit.blocks.has_overflow) { proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); } else { // Allocate only a correct amount of memory for each polynomial @@ -340,14 +339,14 @@ template class DeciderProvingKey_ { requires IsGoblinFlavor; public: - static void move_structured_trace_overflow_to_miscellaneous_block(Circuit& circuit) + static void move_structured_trace_overflow_to_overflow_block(Circuit& circuit) { auto& blocks = circuit.blocks; - auto& misc_block = circuit.blocks.miscellaneous; + auto& overflow_block = circuit.blocks.overflow; // WORKTODO: need to think about what cases to support here. Can't be surprised by an overflow in the IVC // setting but need to be able to determine the overflow for a one-off circuit. - if (misc_block.get_fixed_size() > 0) { + if (overflow_block.get_fixed_size() > 0) { info("Miscellaneous block has non-zero size; setting has_overflow to TRUE."); blocks.has_overflow = true; } @@ -357,23 +356,24 @@ template class DeciderProvingKey_ { for (auto& block : blocks.get()) { size_t block_size = block.size(); size_t fixed_block_size = block.get_fixed_size(); - if (block_size > fixed_block_size && block != misc_block) { + if (block_size > fixed_block_size && block != overflow_block) { // We dont handle this case ASSERT(!block.is_pub_inputs); - // Set flag indicating that at least one block exceeds capacity and the misc block is in use + // Set flag indicating that at least one block exceeds capacity and the overflow block is in use blocks.has_overflow = true; // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to - // account for moving them into the miscellaneous block + // account for moving them into the overflow block if (block.has_ram_rom) { // for (auto& val : block.q_aux()) { // info("q_aux = ", val); // } - uint32_t misc_block_start_idx = misc_block.trace_offset + static_cast(misc_block.size()); - misc_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later - uint32_t offset = misc_block_start_idx + 1; // +1 accounts for duplication of final gate + uint32_t overflow_block_start_idx = + overflow_block.trace_offset + static_cast(overflow_block.size()); + overflow_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later + uint32_t offset = overflow_block_start_idx + 1; // +1 accounts for duplication of final gate bool last_gate_is_memory_read = false; bool last_gate_is_memory_write = false; for (auto& idx : circuit.memory_read_records) { @@ -383,7 +383,7 @@ template class DeciderProvingKey_ { } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero - idx += offset; // shift to correct location in misc block + idx += offset; // shift to correct location in overflow block } } for (auto& idx : circuit.memory_write_records) { @@ -393,30 +393,30 @@ template class DeciderProvingKey_ { } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero - idx += offset; // shift to correct location in misc block + idx += offset; // shift to correct location in overflow block } } - // The last gate in the main block will be duplicated in the miscellaneous block. If it was a memory - // read/write, add the corresponding the misc block index to the memory record indices. + // The last gate in the main block will be duplicated in the overflow block. If it was a memory + // read/write, add the corresponding the overflow block index to the memory record indices. if (last_gate_is_memory_read) { - circuit.memory_read_records.push_back(misc_block_start_idx); + circuit.memory_read_records.push_back(overflow_block_start_idx); } else if (last_gate_is_memory_write) { - circuit.memory_write_records.push_back(misc_block_start_idx); + circuit.memory_write_records.push_back(overflow_block_start_idx); } } - // Move the excess wire and selector data from the offending block to the miscellaneous block + // Move the excess wire and selector data from the offending block to the overflow block size_t start = fixed_block_size - 1; // the final gate in the main block is duplicated size_t end = block_size; - for (auto [wire, misc_wire] : zip_view(block.wires, misc_block.wires)) { + for (auto [wire, overflow_wire] : zip_view(block.wires, overflow_block.wires)) { for (size_t i = start; i < end; ++i) { - misc_wire.emplace_back(wire[i]); + overflow_wire.emplace_back(wire[i]); } wire.resize(fixed_block_size); } - for (auto [selector, misc_selector] : zip_view(block.selectors, misc_block.selectors)) { + for (auto [selector, overflow_selector] : zip_view(block.selectors, overflow_block.selectors)) { for (size_t i = start; i < end; ++i) { - misc_selector.emplace_back(selector[i]); + overflow_selector.emplace_back(selector[i]); } selector.resize(fixed_block_size); } @@ -427,17 +427,17 @@ template class DeciderProvingKey_ { } } // WORKTODO: probably don't set this here but this means it needs to be set correctly on input - // Set the fixed size of the miscellaneous block to its current size - if (misc_block.size() > misc_block.get_fixed_size()) { + // Set the fixed size of the overflow block to its current size + if (overflow_block.size() > overflow_block.get_fixed_size()) { info("WARNING: Overflow value was set too low! Miscellaneous block fixed size: ", - misc_block.get_fixed_size(), + overflow_block.get_fixed_size(), ". Miscellaneous block actual size: ", - misc_block.size()); - // Cannot currently support dynamic resizing of the miscellaneous block since this would require dynamic + overflow_block.size()); + // Cannot currently support dynamic resizing of the overflow block since this would require dynamic // expansion of the accumulator in the IVC setting. // ASSERT(false); - info("Setting miscellaneous block size to: ", misc_block.size()); - misc_block.set_fixed_size(static_cast(misc_block.size())); + info("Setting overflow block size to: ", overflow_block.size()); + overflow_block.set_fixed_size(static_cast(overflow_block.size())); } // WORKTODO: make this spit out a clear display of the original block sizes vs fixed sizes From db0646132dace369c5744000b9f13823be56d894 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 18:11:24 +0000 Subject: [PATCH 13/25] cleanup pass --- .../cpp/src/barretenberg/client_ivc/client_ivc.cpp | 2 +- .../cpp/src/barretenberg/client_ivc/client_ivc.hpp | 2 +- .../src/barretenberg/execution_trace/execution_trace.cpp | 2 -- .../arithmetization/arithmetization.hpp | 2 +- .../arithmetization/mega_arithmetization.hpp | 2 +- .../cpp/src/barretenberg/ultra_honk/CMakeLists.txt | 2 +- .../src/barretenberg/ultra_honk/decider_proving_key.hpp | 2 +- .../src/barretenberg/ultra_honk/mega_composer.test.cpp | 9 +++++++-- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp index dbd86081e0c..1cbfd9b0805 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp @@ -173,7 +173,7 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr proving_key; if (!initialized) { - proving_key = std::make_shared(circuit, trace_settings, /*commitment_key=*/nullptr); + proving_key = std::make_shared(circuit, trace_settings); trace_usage_tracker = ExecutionTraceUsageTracker(trace_settings); } else { proving_key = std::make_shared( diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp index e7b6d904c69..2baf842e13c 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp @@ -115,7 +115,7 @@ class ClientIVC { // Management of linking databus commitments between circuits in the IVC DataBusDepot bus_depot; - // A flag indicating whether or not to construct a structured trace in the DeciderProvingKey + // Settings related to the use of fixed block sizes for each gate in the execution trace TraceSettings trace_settings; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): eventually do away with this. diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index fb3ba32d91f..de3f071dca4 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -143,8 +143,6 @@ typename ExecutionTrace_::TraceData ExecutionTrace_::construct_t // Store the offset of the block containing RAM/ROM read/write gates for use in updating memory records if (block.has_ram_rom) { - // WORKTODO: uh oh. is this a problem if now the misc block can contain aux stuff too? what updates are - // needed? trace_data.ram_rom_offset = offset; } // Store offset of public inputs block for use in the pub input mechanism of the permutation argument diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index 52fdf03a0dc..5a0e8fc7304 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -34,7 +34,7 @@ enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_F struct TraceSettings { TraceStructure structure = TraceStructure::NONE; - uint32_t overflow = 0; + uint32_t overflow = 0; // the size of the overflow block }; /** diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index a12a01033fd..e33305f236b 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -195,7 +195,7 @@ template class MegaArith { this->poseidon2_external = 30128; this->poseidon2_internal = 172000; this->lookup = 200000; - this->overflow = 0; + this->overflow = 0; // can contain arbitrary gate types } }; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt index d1fdb5bc8e0..7c55050a138 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(ultra_honk sumcheck circuit_checker) \ No newline at end of file +barretenberg_module(ultra_honk sumcheck) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 853975ef0e0..d8834c67778 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -355,7 +355,7 @@ template class DeciderProvingKey_ { for (auto& block : blocks.get()) { size_t block_size = block.size(); - size_t fixed_block_size = block.get_fixed_size(); + uint32_t fixed_block_size = block.get_fixed_size(); if (block_size > fixed_block_size && block != overflow_block) { // We dont handle this case ASSERT(!block.is_pub_inputs); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index 56662766f6e..4d25b5a2881 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -225,9 +225,11 @@ TEST_F(MegaHonkTests, MiscellaneousBlockSimple) // Construct and verify Honk proof using a structured trace TraceSettings trace_settings{ TraceStructure::TINY_TEST }; auto proving_key = std::make_shared>(builder, trace_settings); - // builder.blocks.summarize(); + + // We expect that the circuit has overflowed the provided structured trace + EXPECT_TRUE(builder.blocks.has_overflow); + MegaProver prover(proving_key); - // builder.blocks.summarize(); auto verification_key = std::make_shared(proving_key->proving_key); MegaVerifier verifier(verification_key); auto proof = prover.construct_proof(); @@ -251,6 +253,9 @@ TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) // Construct and verify Honk proof using a structured trace auto proving_key = std::make_shared>(builder, trace_settings); + // We expect that the circuit has overflowed the provided structured trace + EXPECT_TRUE(builder.blocks.has_overflow); + // builder.blocks.summarize(); MegaProver prover(proving_key); // builder.blocks.summarize(); From 29e5ce292b4e3203da06c904a2308462da95dfbb Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 19:05:57 +0000 Subject: [PATCH 14/25] more clean and reorg --- .../arithmetization/mega_arithmetization.hpp | 6 +- .../stdlib_circuit_builders/mock_circuits.hpp | 2 +- .../ultra_honk/decider_proving_key.cpp | 94 ++++++++++++++ .../ultra_honk/decider_proving_key.hpp | 115 +----------------- 4 files changed, 99 insertions(+), 118 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index e33305f236b..5ef803a39b5 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -35,7 +35,7 @@ template class MegaArith { T poseidon2_external; T poseidon2_internal; T lookup; - T overflow; + T overflow; // block gates of arbitrary type that overflow their designated block auto get() { @@ -195,7 +195,7 @@ template class MegaArith { this->poseidon2_external = 30128; this->poseidon2_internal = 172000; this->lookup = 200000; - this->overflow = 0; // can contain arbitrary gate types + this->overflow = 0; } }; @@ -271,7 +271,7 @@ template class MegaArith { struct TraceBlocks : public MegaTraceBlocks { - bool has_overflow = false; + bool has_overflow = false; // indicates whether the overflow block has non-zero fixed or actual size TraceBlocks() { diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp index 0a683a187f0..070046f3f15 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp @@ -89,7 +89,7 @@ class MockCircuits { /** * @brief Add some simple RAM (aux) gates for testing memory read/write functionality - * @brief Each iteration adds + * @brief Each iteration adds 18 gates (including finalization) * * @param builder */ diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index cac01ed0c47..8bea030ed33 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -80,6 +80,100 @@ void DeciderProvingKey_::construct_databus_polynomials(Circuit& circuit) } } +template +void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_block(Circuit& circuit) +{ + auto& blocks = circuit.blocks; + auto& overflow_block = circuit.blocks.overflow; + + // Set has_overflow to true if a nonzero fixed size has been prescribed for the overflow block + blocks.has_overflow = (overflow_block.get_fixed_size() > 0); + + blocks.compute_offsets(/*is_structured=*/true); + + for (auto& block : blocks.get()) { + size_t block_size = block.size(); + uint32_t fixed_block_size = block.get_fixed_size(); + if (block_size > fixed_block_size && block != overflow_block) { + // We dont handle this case + ASSERT(!block.is_pub_inputs); + + // Set flag indicating that at least one block exceeds capacity and the overflow block is in use + blocks.has_overflow = true; + + // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to + // account for moving them into the overflow block + if (block.has_ram_rom) { + + uint32_t overflow_block_start_idx = + overflow_block.trace_offset + static_cast(overflow_block.size()); + overflow_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later + uint32_t offset = overflow_block_start_idx + 1; // +1 accounts for duplication of final gate + bool last_gate_is_memory_read = false; + bool last_gate_is_memory_write = false; + for (auto& idx : circuit.memory_read_records) { + if (idx == fixed_block_size - 1) { + last_gate_is_memory_read = true; + // info("last_gate_is_memory_read."); + } + if (idx >= fixed_block_size) { + idx -= fixed_block_size; // redefine index from zero + idx += offset; // shift to correct location in overflow block + } + } + for (auto& idx : circuit.memory_write_records) { + if (idx == fixed_block_size - 1) { + last_gate_is_memory_write = true; + // info("last_gate_is_memory_write."); + } + if (idx >= fixed_block_size) { + idx -= fixed_block_size; // redefine index from zero + idx += offset; // shift to correct location in overflow block + } + } + // The last gate in the main block will be duplicated in the overflow block. If it was a memory + // read/write, add the corresponding the overflow block index to the memory record indices. + if (last_gate_is_memory_read) { + circuit.memory_read_records.push_back(overflow_block_start_idx); + } else if (last_gate_is_memory_write) { + circuit.memory_write_records.push_back(overflow_block_start_idx); + } + } + + // Move the excess wire and selector data from the offending block to the overflow block + size_t start = fixed_block_size - 1; // the final gate in the main block is duplicated + size_t end = block_size; + for (auto [wire, overflow_wire] : zip_view(block.wires, overflow_block.wires)) { + for (size_t i = start; i < end; ++i) { + overflow_wire.emplace_back(wire[i]); + } + wire.resize(fixed_block_size); + } + for (auto [selector, overflow_selector] : zip_view(block.selectors, overflow_block.selectors)) { + for (size_t i = start; i < end; ++i) { + overflow_selector.emplace_back(selector[i]); + } + selector.resize(fixed_block_size); + } + // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors + for (auto& selector : block.get_gate_selectors()) { + selector.back() = 0; + } + } + } + + // Set the fixed size of the overflow block to its current size + if (overflow_block.size() > overflow_block.get_fixed_size()) { + info("WARNING: Structured trace overflowed beyond the prescribed fixed overflow size. This is valid in the " + "context of VK computation but not in the IVC setting. \nFixed overflow size: ", + overflow_block.get_fixed_size(), + ". \nActual overflow size: ", + overflow_block.size()); + overflow_block.set_fixed_size(static_cast(overflow_block.size())); + blocks.summarize(); + } +} + template class DeciderProvingKey_; template class DeciderProvingKey_; template class DeciderProvingKey_; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index d8834c67778..1409e5161de 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -149,10 +149,6 @@ template class DeciderProvingKey_ { } if constexpr (HasDataBus) { - // WORKTODO: could be this: - // for (auto& poly : proving_key.polynomials.get_databus_entities()) { - // poly = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); - // } proving_key.polynomials.calldata = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); proving_key.polynomials.calldata_read_counts = Polynomial(MAX_DATABUS_SIZE, proving_key.circuit_size); @@ -338,116 +334,7 @@ template class DeciderProvingKey_ { void construct_databus_polynomials(Circuit&) requires IsGoblinFlavor; - public: - static void move_structured_trace_overflow_to_overflow_block(Circuit& circuit) - { - auto& blocks = circuit.blocks; - auto& overflow_block = circuit.blocks.overflow; - - // WORKTODO: need to think about what cases to support here. Can't be surprised by an overflow in the IVC - // setting but need to be able to determine the overflow for a one-off circuit. - if (overflow_block.get_fixed_size() > 0) { - info("Miscellaneous block has non-zero size; setting has_overflow to TRUE."); - blocks.has_overflow = true; - } - - blocks.compute_offsets(/*is_structured=*/true); - - for (auto& block : blocks.get()) { - size_t block_size = block.size(); - uint32_t fixed_block_size = block.get_fixed_size(); - if (block_size > fixed_block_size && block != overflow_block) { - // We dont handle this case - ASSERT(!block.is_pub_inputs); - - // Set flag indicating that at least one block exceeds capacity and the overflow block is in use - blocks.has_overflow = true; - - // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to - // account for moving them into the overflow block - if (block.has_ram_rom) { - - // for (auto& val : block.q_aux()) { - // info("q_aux = ", val); - // } - uint32_t overflow_block_start_idx = - overflow_block.trace_offset + static_cast(overflow_block.size()); - overflow_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later - uint32_t offset = overflow_block_start_idx + 1; // +1 accounts for duplication of final gate - bool last_gate_is_memory_read = false; - bool last_gate_is_memory_write = false; - for (auto& idx : circuit.memory_read_records) { - if (idx == fixed_block_size - 1) { - last_gate_is_memory_read = true; - // info("last_gate_is_memory_read."); - } - if (idx >= fixed_block_size) { - idx -= fixed_block_size; // redefine index from zero - idx += offset; // shift to correct location in overflow block - } - } - for (auto& idx : circuit.memory_write_records) { - if (idx == fixed_block_size - 1) { - last_gate_is_memory_write = true; - // info("last_gate_is_memory_write."); - } - if (idx >= fixed_block_size) { - idx -= fixed_block_size; // redefine index from zero - idx += offset; // shift to correct location in overflow block - } - } - // The last gate in the main block will be duplicated in the overflow block. If it was a memory - // read/write, add the corresponding the overflow block index to the memory record indices. - if (last_gate_is_memory_read) { - circuit.memory_read_records.push_back(overflow_block_start_idx); - } else if (last_gate_is_memory_write) { - circuit.memory_write_records.push_back(overflow_block_start_idx); - } - } - - // Move the excess wire and selector data from the offending block to the overflow block - size_t start = fixed_block_size - 1; // the final gate in the main block is duplicated - size_t end = block_size; - for (auto [wire, overflow_wire] : zip_view(block.wires, overflow_block.wires)) { - for (size_t i = start; i < end; ++i) { - overflow_wire.emplace_back(wire[i]); - } - wire.resize(fixed_block_size); - } - for (auto [selector, overflow_selector] : zip_view(block.selectors, overflow_block.selectors)) { - for (size_t i = start; i < end; ++i) { - overflow_selector.emplace_back(selector[i]); - } - selector.resize(fixed_block_size); - } - // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors - for (auto& selector : block.get_gate_selectors()) { - selector.back() = 0; - } - } - } - // WORKTODO: probably don't set this here but this means it needs to be set correctly on input - // Set the fixed size of the overflow block to its current size - if (overflow_block.size() > overflow_block.get_fixed_size()) { - info("WARNING: Overflow value was set too low! Miscellaneous block fixed size: ", - overflow_block.get_fixed_size(), - ". Miscellaneous block actual size: ", - overflow_block.size()); - // Cannot currently support dynamic resizing of the overflow block since this would require dynamic - // expansion of the accumulator in the IVC setting. - // ASSERT(false); - info("Setting overflow block size to: ", overflow_block.size()); - overflow_block.set_fixed_size(static_cast(overflow_block.size())); - } - - // WORKTODO: make this spit out a clear display of the original block sizes vs fixed sizes - if (blocks.has_overflow) { - info("WARNING: At least one gate type exceeds the limit defined in the structured trace. The overflow " - "block is in use and may negatively impact performance of the prover!"); - blocks.summarize(); - // ASSERT(false); - } - } + static void move_structured_trace_overflow_to_overflow_block(Circuit& circuit); }; } // namespace bb From 6e2a49ffaa0b8e8bc19a3d95726746e76265bb48 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 20:08:33 +0000 Subject: [PATCH 15/25] add megaHonk test for various types of overflow --- .../cpp/src/barretenberg/constants.hpp | 2 - .../arithmetization/mega_arithmetization.hpp | 28 +----- .../ultra_honk/decider_proving_key.cpp | 8 +- .../ultra_honk/mega_composer.test.cpp | 85 +++++++++++-------- 4 files changed, 57 insertions(+), 66 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/constants.hpp b/barretenberg/cpp/src/barretenberg/constants.hpp index bd78d7626db..696eecc0aa5 100644 --- a/barretenberg/cpp/src/barretenberg/constants.hpp +++ b/barretenberg/cpp/src/barretenberg/constants.hpp @@ -13,9 +13,7 @@ static constexpr uint32_t CONST_PG_LOG_N = 20; static constexpr uint32_t CONST_ECCVM_LOG_N = 16; -// static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 8; // DEBUG! static constexpr uint32_t MAX_LOOKUP_TABLES_SIZE = 70000; -// static constexpr uint32_t MAX_DATABUS_SIZE = 3; // DEBUG! static constexpr uint32_t MAX_DATABUS_SIZE = 10000; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index 5ef803a39b5..e03d61b37c0 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -79,37 +79,13 @@ template class MegaArith { struct TinyTestStructuredBlockSizes : public MegaTraceBlocks { TinyTestStructuredBlockSizes() { - // const uint32_t FIXED_SIZE = 1 << 14; - // this->ecc_op = FIXED_SIZE; - // this->pub_inputs = FIXED_SIZE; - // this->busread = FIXED_SIZE; - // this->arithmetic = 1 << 15; - // this->delta_range = FIXED_SIZE; - // this->elliptic = FIXED_SIZE; - // this->aux = FIXED_SIZE; - // this->poseidon2_external = FIXED_SIZE; - // this->poseidon2_internal = 1 << 15; - // this->lookup = FIXED_SIZE; - // this->overflow = 0; - - // this->ecc_op = 50; - // this->pub_inputs = 33; - // this->busread = 3; - // this->arithmetic = 3417; // 3418 - // this->delta_range = 2002; - // this->elliptic = 2; - // this->aux = 26; - // this->poseidon2_external = 812; - // this->poseidon2_internal = 4619; - // this->lookup = 2; - this->ecc_op = 18; this->pub_inputs = 1; this->busread = 3; - this->arithmetic = 15; // 15 + this->arithmetic = 1 << 14; this->delta_range = 5; this->elliptic = 2; - this->aux = 10; // 20 + this->aux = 10; this->poseidon2_external = 2; this->poseidon2_internal = 2; this->lookup = 2; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index 8bea030ed33..1c4e0178c15 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -80,6 +80,12 @@ void DeciderProvingKey_::construct_databus_polynomials(Circuit& circuit) } } +/** + * @brief WORKTODO + * + * @tparam Flavor + * @param circuit + */ template void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_block(Circuit& circuit) { @@ -165,7 +171,7 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc // Set the fixed size of the overflow block to its current size if (overflow_block.size() > overflow_block.get_fixed_size()) { info("WARNING: Structured trace overflowed beyond the prescribed fixed overflow size. This is valid in the " - "context of VK computation but not in the IVC setting. \nFixed overflow size: ", + "context of one-off VK/proof generation but not in the IVC setting. \nPrescribed overflow size: ", overflow_block.get_fixed_size(), ". \nActual overflow size: ", overflow_block.size()); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp index 4d25b5a2881..da4f03742c2 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_composer.test.cpp @@ -212,55 +212,66 @@ TEST_F(MegaHonkTests, MultipleCircuitsHonkAndMerge) } /** - * @brief Test proof construction/verification for a structured execution trace + * @brief Test the structured trace overflow mechanism for various circuits which overflow in different ways * */ -TEST_F(MegaHonkTests, MiscellaneousBlockSimple) +TEST_F(MegaHonkTests, StructuredTraceOverflow) { - MegaCircuitBuilder builder; + using ProvingKey = DeciderProvingKey_; + using VerificationKey = MegaFlavor::VerificationKey; + using Builder = MegaCircuitBuilder; - GoblinMockCircuits::construct_simple_circuit(builder); - MockCircuits::add_arithmetic_gates(builder, 8); + // Construct and verify a MegaHonk proof using a structured trace + auto construct_and_verify_proof_with_structured_trace = [](Builder& builder, TraceSettings& trace_settings) { + auto proving_key = std::make_shared(builder, trace_settings); + + // We expect that the circuit has overflowed the provided structured trace + EXPECT_TRUE(builder.blocks.has_overflow); + + MegaProver prover(proving_key); + auto verification_key = std::make_shared(proving_key->proving_key); + MegaVerifier verifier(verification_key); + auto proof = prover.construct_proof(); + EXPECT_TRUE(verifier.verify_proof(proof)); + }; - // Construct and verify Honk proof using a structured trace TraceSettings trace_settings{ TraceStructure::TINY_TEST }; - auto proving_key = std::make_shared>(builder, trace_settings); - // We expect that the circuit has overflowed the provided structured trace - EXPECT_TRUE(builder.blocks.has_overflow); + { // Overflow in Arithmetic block only + Builder builder; - MegaProver prover(proving_key); - auto verification_key = std::make_shared(proving_key->proving_key); - MegaVerifier verifier(verification_key); - auto proof = prover.construct_proof(); - EXPECT_TRUE(verifier.verify_proof(proof)); -} + GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_arithmetic_gates(builder, 1 << 15); -/** - * @brief Test proof construction/verification for a structured execution trace - * - */ -TEST_F(MegaHonkTests, MiscellaneousBlockAuxOverflow) -{ - MegaCircuitBuilder builder; + construct_and_verify_proof_with_structured_trace(builder, trace_settings); + } - TraceSettings trace_settings{ TraceStructure::TINY_TEST }; + { // Overflow in Aux block (RAM gates; uses memory records which requires specific logic in overflow mechanism) + Builder builder; - GoblinMockCircuits::construct_simple_circuit(builder); - MockCircuits::add_arithmetic_gates(builder, 8); - MockCircuits::add_RAM_gates(builder); + GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_RAM_gates(builder); - // Construct and verify Honk proof using a structured trace - auto proving_key = std::make_shared>(builder, trace_settings); + construct_and_verify_proof_with_structured_trace(builder, trace_settings); + } - // We expect that the circuit has overflowed the provided structured trace - EXPECT_TRUE(builder.blocks.has_overflow); + { // Overflow in Lookup block only + Builder builder; - // builder.blocks.summarize(); - MegaProver prover(proving_key); - // builder.blocks.summarize(); - auto verification_key = std::make_shared(proving_key->proving_key); - MegaVerifier verifier(verification_key); - auto proof = prover.construct_proof(); - EXPECT_TRUE(verifier.verify_proof(proof)); + GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_lookup_gates(builder, /*num_iterations=*/8); + + construct_and_verify_proof_with_structured_trace(builder, trace_settings); + } + + { // Overflow in Multiple blocks simultaneously + Builder builder; + + GoblinMockCircuits::construct_simple_circuit(builder); + MockCircuits::add_arithmetic_gates(builder, 1 << 15); + MockCircuits::add_RAM_gates(builder); + MockCircuits::add_lookup_gates(builder, /*num_iterations=*/8); + + construct_and_verify_proof_with_structured_trace(builder, trace_settings); + } } From ea81b181d498f7d49b4ce91ea48501b4efed7907 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 21:19:17 +0000 Subject: [PATCH 16/25] update IVC test --- .../src/barretenberg/client_ivc/client_ivc.test.cpp | 13 +++++++++---- .../arithmetization/arithmetization.hpp | 2 +- .../arithmetization/mega_arithmetization.hpp | 2 +- .../arithmetization/ultra_arithmetization.hpp | 2 +- .../barretenberg/ultra_honk/decider_proving_key.cpp | 3 ++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index 1582f2f91db..ab64290a396 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -390,24 +390,29 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) } /** - * @brief Test use of overflow block + * @brief Test use of structured trace overflow block mechanism + * @details Accumulate 4 circuits which have progressively more arithmetic gates. The final two overflow the prescribed + * arithmetic block size and make use of the overflow block. * */ TEST_F(ClientIVCTests, MiscellaneousBlock) { ClientIVC ivc; - ivc.trace_settings = { TraceStructure::SMALL_TEST, /*overflow=*/1 << 17 }; + + // Define trace settings with sufficient overflow capacity to acommodate each of the circuits to be accumulated + uint32_t overflow_capacity = 1 << 17; + ivc.trace_settings = { TraceStructure::SMALL_TEST, overflow_capacity }; MockCircuitProducer circuit_producer; - size_t NUM_CIRCUITS = 2; + size_t NUM_CIRCUITS = 4; // Construct and accumulate some circuits of varying size size_t log2_num_gates = 14; for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_gates); ivc.accumulate(circuit); - log2_num_gates += 2; + log2_num_gates += 1; } EXPECT_TRUE(ivc.prove_and_verify()); diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index 5a0e8fc7304..28271fb1a22 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -34,7 +34,7 @@ enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_F struct TraceSettings { TraceStructure structure = TraceStructure::NONE; - uint32_t overflow = 0; // the size of the overflow block + uint32_t overflow_capacity = 0; // the size of the overflow block }; /** diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp index e03d61b37c0..7cb03ae310a 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/mega_arithmetization.hpp @@ -280,7 +280,7 @@ template class MegaArith { block.set_fixed_size(size); } // Set the size of overflow block containing the overflow from all other blocks - this->overflow.set_fixed_size(settings.overflow); + this->overflow.set_fixed_size(settings.overflow_capacity); } void compute_offsets(bool is_structured) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp index e2f46c29416..a188d2faef5 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/ultra_arithmetization.hpp @@ -134,7 +134,7 @@ template class UltraArith { for (auto [block, size] : zip_view(this->get(), fixed_block_sizes.get())) { block.set_fixed_size(size); } - this->overflow.set_fixed_size(settings.overflow); + this->overflow.set_fixed_size(settings.overflow_capacity); } void compute_offsets(bool is_structured) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index 1c4e0178c15..5853cab2465 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -174,7 +174,8 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc "context of one-off VK/proof generation but not in the IVC setting. \nPrescribed overflow size: ", overflow_block.get_fixed_size(), ". \nActual overflow size: ", - overflow_block.size()); + overflow_block.size(), + "\n"); overflow_block.set_fixed_size(static_cast(overflow_block.size())); blocks.summarize(); } From 9e35acf363c53b4cb644a1be8f846d68e16a5a45 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 22:00:44 +0000 Subject: [PATCH 17/25] comments --- .../ultra_honk/decider_proving_key.cpp | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index 5853cab2465..fd76b17d5cf 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -81,7 +81,17 @@ void DeciderProvingKey_::construct_databus_polynomials(Circuit& circuit) } /** - * @brief WORKTODO + * @brief Check that the number of gates in each block does not exceed capacity. Move any overflow to the overflow + * block. + * @details Using a structured trace (fixed capcity for each gate type) optimizes the efficiency of folding. However, + * to acommodate circuits which cannot fit into a prescribed trace, gates which overflow their corresponding block are + * placed into an overflow block which can contain arbitrary gate types. + * @note One sublety is that gates at row i may in general utilize the values at row i+1 via shifts. If the last row in + * a full-capactiy block is such a gate, then moving the overflow out of sequence will cause that gate not to be + * satisfied. To avoid this, when a block overflows, the final gate in the block is duplicated, once as a dummy gate in + * the main block (so that the prior gate can read into it but it does not itself try to read into the next row) and + * again as a normal gate in the overflow block. Therefore, the total number of gates in the circuit increases by one + * for each block that overflows. * * @tparam Flavor * @param circuit @@ -95,7 +105,7 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc // Set has_overflow to true if a nonzero fixed size has been prescribed for the overflow block blocks.has_overflow = (overflow_block.get_fixed_size() > 0); - blocks.compute_offsets(/*is_structured=*/true); + blocks.compute_offsets(/*is_structured=*/true); // compute the offset of each fixed size block for (auto& block : blocks.get()) { size_t block_size = block.size(); @@ -104,23 +114,21 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc // We dont handle this case ASSERT(!block.is_pub_inputs); - // Set flag indicating that at least one block exceeds capacity and the overflow block is in use + // Set has_overflow to true if at least one block exceeds its capacity blocks.has_overflow = true; - // if the overflowing block contains RAM/ROM, need to update gate indices in memory records to - // account for moving them into the overflow block + // The circuit memory records store the indices at which a RAM/ROM read/write has occurred. If the block + // containing RAM/ROM gates overflows, these indices of the corresponding gates need to be updated to + // reflect their new position in the overflow block if (block.has_ram_rom) { - uint32_t overflow_block_start_idx = - overflow_block.trace_offset + static_cast(overflow_block.size()); - overflow_block_start_idx -= block.trace_offset; // we'll add block.trace_offset to everything later - uint32_t offset = overflow_block_start_idx + 1; // +1 accounts for duplication of final gate - bool last_gate_is_memory_read = false; - bool last_gate_is_memory_write = false; + uint32_t overflow_cur_idx = overflow_block.trace_offset + static_cast(overflow_block.size()); + overflow_cur_idx -= block.trace_offset; // we'll add block.trace_offset to everything later + uint32_t offset = overflow_cur_idx + 1; // +1 accounts for duplication of final gate for (auto& idx : circuit.memory_read_records) { + // last gate in the main block will be duplicated; if necessary, duplicate the memory read idx too if (idx == fixed_block_size - 1) { - last_gate_is_memory_read = true; - // info("last_gate_is_memory_read."); + circuit.memory_read_records.push_back(overflow_cur_idx); } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero @@ -128,40 +136,34 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc } } for (auto& idx : circuit.memory_write_records) { + // last gate in the main block will be duplicated; if necessary, duplicate the memory write idx too if (idx == fixed_block_size - 1) { - last_gate_is_memory_write = true; - // info("last_gate_is_memory_write."); + circuit.memory_write_records.push_back(overflow_cur_idx); } if (idx >= fixed_block_size) { idx -= fixed_block_size; // redefine index from zero idx += offset; // shift to correct location in overflow block } } - // The last gate in the main block will be duplicated in the overflow block. If it was a memory - // read/write, add the corresponding the overflow block index to the memory record indices. - if (last_gate_is_memory_read) { - circuit.memory_read_records.push_back(overflow_block_start_idx); - } else if (last_gate_is_memory_write) { - circuit.memory_write_records.push_back(overflow_block_start_idx); - } } // Move the excess wire and selector data from the offending block to the overflow block - size_t start = fixed_block_size - 1; // the final gate in the main block is duplicated - size_t end = block_size; + size_t overflow_start = fixed_block_size - 1; // the final gate in the main block is duplicated + size_t overflow_end = block_size; for (auto [wire, overflow_wire] : zip_view(block.wires, overflow_block.wires)) { - for (size_t i = start; i < end; ++i) { + for (size_t i = overflow_start; i < overflow_end; ++i) { overflow_wire.emplace_back(wire[i]); } - wire.resize(fixed_block_size); + wire.resize(fixed_block_size); // shrink the main block to its max capacity } for (auto [selector, overflow_selector] : zip_view(block.selectors, overflow_block.selectors)) { - for (size_t i = start; i < end; ++i) { + for (size_t i = overflow_start; i < overflow_end; ++i) { overflow_selector.emplace_back(selector[i]); } - selector.resize(fixed_block_size); + selector.resize(fixed_block_size); // shrink the main block to its max capacity } - // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors + // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors so ensure it + // does not erroneously read into the next row. (The active duplicate exists in the overflow block). for (auto& selector : block.get_gate_selectors()) { selector.back() = 0; } From 02ba4bbafc36e260e20888dbef8f563562d418ee Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 22:04:10 +0000 Subject: [PATCH 18/25] test name update --- .../cpp/src/barretenberg/client_ivc/client_ivc.test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index ab64290a396..7d3ed2e78ad 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -392,10 +392,10 @@ TEST(ClientIVCBenchValidation, Full6MockedVKs) /** * @brief Test use of structured trace overflow block mechanism * @details Accumulate 4 circuits which have progressively more arithmetic gates. The final two overflow the prescribed - * arithmetic block size and make use of the overflow block. + * arithmetic block size and make use of the overflow block which has sufficient capacity. * */ -TEST_F(ClientIVCTests, MiscellaneousBlock) +TEST_F(ClientIVCTests, StructuredTraceOverflow) { ClientIVC ivc; From b014c9e53dc1ab9e857c47df0d9caa478d12a08e Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 7 Nov 2024 22:39:53 +0000 Subject: [PATCH 19/25] comments and clean --- .../client_ivc/client_ivc.test.cpp | 13 +++++------ .../client_ivc_integration.test.cpp | 2 +- .../client_ivc/mock_circuit_producer.hpp | 4 ++-- .../execution_trace/execution_trace.cpp | 9 -------- .../ultra_honk/decider_proving_key.cpp | 23 ++++++++++--------- .../barretenberg/ultra_honk/oink_prover.cpp | 22 ------------------ 6 files changed, 21 insertions(+), 52 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp index 7d3ed2e78ad..f61a6d786b3 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp @@ -76,11 +76,11 @@ class ClientIVCTests : public ::testing::Test { } auto precompute_verification_keys(const size_t num_circuits, - TraceStructure trace_structure, + TraceSettings trace_settings, size_t log2_num_gates = 16) { ClientIVC ivc; // temporary IVC instance needed to produce the complete kernel circuits - ivc.trace_settings.structure = trace_structure; + ivc.trace_settings = trace_settings; std::vector> vkeys; @@ -307,7 +307,7 @@ TEST_F(ClientIVCTests, PrecomputedVerificationKeys) MockCircuitProducer circuit_producer; - auto precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, TraceStructure::NONE); + auto precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, TraceSettings{}); // Construct and accumulate set of circuits using the precomputed vkeys for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { @@ -333,7 +333,7 @@ TEST_F(ClientIVCTests, StructuredPrecomputedVKs) MockCircuitProducer circuit_producer; auto precomputed_vks = - circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings.structure, log2_num_gates); + circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings, log2_num_gates); // Construct and accumulate set of circuits using the precomputed vkeys for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) { @@ -360,8 +360,7 @@ TEST(ClientIVCBenchValidation, Full6) ivc.trace_settings.structure = TraceStructure::CLIENT_IVC_BENCH; size_t total_num_circuits{ 12 }; PrivateFunctionExecutionMockCircuitProducer circuit_producer; - auto precomputed_vkeys = - circuit_producer.precompute_verification_keys(total_num_circuits, ivc.trace_settings.structure); + auto precomputed_vkeys = circuit_producer.precompute_verification_keys(total_num_circuits, ivc.trace_settings); perform_ivc_accumulation_rounds(total_num_circuits, ivc, precomputed_vkeys); auto proof = ivc.prove(); bool verified = verify_ivc(proof, ivc); @@ -399,7 +398,7 @@ TEST_F(ClientIVCTests, StructuredTraceOverflow) { ClientIVC ivc; - // Define trace settings with sufficient overflow capacity to acommodate each of the circuits to be accumulated + // Define trace settings with sufficient overflow capacity to accommodate each of the circuits to be accumulated uint32_t overflow_capacity = 1 << 17; ivc.trace_settings = { TraceStructure::SMALL_TEST, overflow_capacity }; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp index 6f1887287ae..9fed2ce5ca5 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc_integration.test.cpp @@ -97,7 +97,7 @@ TEST_F(ClientIVCIntegrationTests, BenchmarkCasePrecomputedVKs) std::vector> precomputed_vks; { MockCircuitProducer circuit_producer; - precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings.structure); + precomputed_vks = circuit_producer.precompute_verification_keys(NUM_CIRCUITS, ivc.trace_settings); } MockCircuitProducer circuit_producer; diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp index 8fec8712943..eae4966e32a 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp @@ -135,10 +135,10 @@ class PrivateFunctionExecutionMockCircuitProducer { * @param trace_structure Trace structuring must be known in advance because it effects the VKs * @return set of num_circuits-many verification keys */ - auto precompute_verification_keys(const size_t num_circuits, TraceStructure trace_structure) + auto precompute_verification_keys(const size_t num_circuits, TraceSettings trace_settings) { ClientIVC ivc; // temporary IVC instance needed to produce the complete kernel circuits - ivc.trace_settings.structure = trace_structure; + ivc.trace_settings = trace_settings; std::vector> vkeys; diff --git a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp index de3f071dca4..93945d1907e 100644 --- a/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp +++ b/barretenberg/cpp/src/barretenberg/execution_trace/execution_trace.cpp @@ -76,15 +76,6 @@ void ExecutionTrace_::add_memory_records_to_proving_key(TraceData& trace for (auto& index : builder.memory_write_records) { proving_key.memory_write_records.emplace_back(index + trace_data.ram_rom_offset); } - // info("trace_data.ram_rom_offset = ", trace_data.ram_rom_offset); - // info("READ records: "); - // for (auto idx : proving_key.memory_read_records) { - // info("idx = ", idx); - // } - // info("WRITE records: "); - // for (auto idx : proving_key.memory_write_records) { - // info("idx = ", idx); - // } } template diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index fd76b17d5cf..73a8844e460 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -81,13 +81,13 @@ void DeciderProvingKey_::construct_databus_polynomials(Circuit& circuit) } /** - * @brief Check that the number of gates in each block does not exceed capacity. Move any overflow to the overflow - * block. + * @brief Check that the number of gates in each block does not exceed its fixed capacity. Move any overflow to the + * overflow block. * @details Using a structured trace (fixed capcity for each gate type) optimizes the efficiency of folding. However, - * to acommodate circuits which cannot fit into a prescribed trace, gates which overflow their corresponding block are + * to accommodate circuits which cannot fit into a prescribed trace, gates which overflow their corresponding block are * placed into an overflow block which can contain arbitrary gate types. * @note One sublety is that gates at row i may in general utilize the values at row i+1 via shifts. If the last row in - * a full-capactiy block is such a gate, then moving the overflow out of sequence will cause that gate not to be + * a full-capacity block is such a gate, then moving the overflow out of sequence will cause that gate not to be * satisfied. To avoid this, when a block overflows, the final gate in the block is duplicated, once as a dummy gate in * the main block (so that the prior gate can read into it but it does not itself try to read into the next row) and * again as a normal gate in the overflow block. Therefore, the total number of gates in the circuit increases by one @@ -107,6 +107,7 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc blocks.compute_offsets(/*is_structured=*/true); // compute the offset of each fixed size block + // Check each block for capacity overflow; if necessary move gates into the overflow block for (auto& block : blocks.get()) { size_t block_size = block.size(); uint32_t fixed_block_size = block.get_fixed_size(); @@ -117,9 +118,9 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc // Set has_overflow to true if at least one block exceeds its capacity blocks.has_overflow = true; - // The circuit memory records store the indices at which a RAM/ROM read/write has occurred. If the block - // containing RAM/ROM gates overflows, these indices of the corresponding gates need to be updated to - // reflect their new position in the overflow block + // The circuit memory read/write records store the indices at which a RAM/ROM read/write has occurred. If + // the block containing RAM/ROM gates overflows, the indices of the corresponding gates in the memory + // records need to be updated to reflect their new position in the overflow block if (block.has_ram_rom) { uint32_t overflow_cur_idx = overflow_block.trace_offset + static_cast(overflow_block.size()); @@ -152,18 +153,18 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc size_t overflow_end = block_size; for (auto [wire, overflow_wire] : zip_view(block.wires, overflow_block.wires)) { for (size_t i = overflow_start; i < overflow_end; ++i) { - overflow_wire.emplace_back(wire[i]); + overflow_wire.push_back(wire[i]); } wire.resize(fixed_block_size); // shrink the main block to its max capacity } for (auto [selector, overflow_selector] : zip_view(block.selectors, overflow_block.selectors)) { for (size_t i = overflow_start; i < overflow_end; ++i) { - overflow_selector.emplace_back(selector[i]); + overflow_selector.push_back(selector[i]); } selector.resize(fixed_block_size); // shrink the main block to its max capacity } - // Convert the final gate in the main block to a 'dummy' gate by turning off all gate selectors so ensure it - // does not erroneously read into the next row. (The active duplicate exists in the overflow block). + // Convert final gate in the main block to a 'dummy' gate by turning off all gate selectors. This ensures it + // does not erroneously read into the next row. (An 'active' duplicate exists in the overflow block). for (auto& selector : block.get_gate_selectors()) { selector.back() = 0; } diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index 830b0a5f264..b9279d2a5ae 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -160,30 +160,8 @@ template void OinkProver::execute_sorted_list_acc proving_key->relation_parameters.eta_two = eta_two; proving_key->relation_parameters.eta_three = eta_three; - // auto& polys = proving_key->proving_key.polynomials; - // std::vector idxs = { 54, 61 }; - // for (auto idx : idxs) { - // info("IDX = ", idx); - // info("q_aux = ", polys.q_aux[idx]); - // info("q_arith = ", polys.q_arith[idx]); - // info("w_l = ", polys.w_l[idx]); - // info("w_r = ", polys.w_r[idx]); - // info("w_o = ", polys.w_o[idx]); - // info("w_4 = ", polys.w_4[idx]); - // } - proving_key->proving_key.add_ram_rom_memory_records_to_wire_4(eta, eta_two, eta_three); - // for (auto idx : idxs) { - // info("IDX = ", idx); - // info("q_aux = ", polys.q_aux[idx]); - // info("q_arith = ", polys.q_arith[idx]); - // info("w_l = ", polys.w_l[idx]); - // info("w_r = ", polys.w_r[idx]); - // info("w_o = ", polys.w_o[idx]); - // info("w_4 = ", polys.w_4[idx]); - // } - // Commit to lookup argument polynomials and the finalized (i.e. with memory records) fourth wire polynomial { PROFILE_THIS_NAME("COMMIT::lookup_counts_tags"); From 93bb09d66eedd147e5deb5852ae32fd703da6a00 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Fri, 8 Nov 2024 14:50:05 +0000 Subject: [PATCH 20/25] comment --- .../plonk_honk_shared/arithmetization/arithmetization.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp index 28271fb1a22..066504bdb40 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/arithmetization/arithmetization.hpp @@ -34,7 +34,9 @@ enum class TraceStructure { NONE, TINY_TEST, SMALL_TEST, CLIENT_IVC_BENCH, E2E_F struct TraceSettings { TraceStructure structure = TraceStructure::NONE; - uint32_t overflow_capacity = 0; // the size of the overflow block + // The size of the overflow block. Specified separately because it is allowed to be determined at runtime in the + // context of VK computation + uint32_t overflow_capacity = 0; }; /** From aedac3414f0d9991f4793a6c25964dddd138c823 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Fri, 8 Nov 2024 15:47:40 +0000 Subject: [PATCH 21/25] only run overflow test w mega --- barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp index e4419dc2dcd..a3515219707 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp @@ -250,7 +250,7 @@ TYPED_TEST(MegaHonkTests, MultipleCircuitsHonkAndMerge) */ TYPED_TEST(MegaHonkTests, StructuredTraceOverflow) { - using Flavor = TypeParam; + using Flavor = MegaFlavor; using Builder = Flavor::CircuitBuilder; TraceSettings trace_settings{ TraceStructure::TINY_TEST }; From 24eaaaf9caa8813e0f4ae607c11721e46c419d7d Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Fri, 8 Nov 2024 15:53:43 +0000 Subject: [PATCH 22/25] update join split circuit hash based on addition of new block --- .../src/barretenberg/examples/join_split/join_split.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp b/barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp index 1ef0dfe3305..f56dbf882e7 100644 --- a/barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp +++ b/barretenberg/cpp/src/barretenberg/examples/join_split/join_split.test.cpp @@ -703,7 +703,7 @@ TEST_F(join_split_tests, test_0_input_notes_and_detect_circuit_change) // The below part detects any changes in the join-split circuit constexpr size_t DYADIC_CIRCUIT_SIZE = 1 << 16; - constexpr uint256_t CIRCUIT_HASH("0x2b30566e4d921ea9b0c76802d86ea5b8381ffa78ef143af1b0d0e3045862cb6b"); + constexpr uint256_t CIRCUIT_HASH("0xe833d0ffaa99f39ca31ef79ded0089b0ec614eb4a528c4e1c4af6221a003fa6e"); const uint256_t circuit_hash = circuit.hash_circuit(); // circuit is finalized now From 6776c4cdd178071cdfd8e86f65729358cd851aa7 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Fri, 8 Nov 2024 17:16:38 +0000 Subject: [PATCH 23/25] actually disable new mega test for zk flavor --- .../cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp index a3515219707..188b6bde3b4 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp @@ -50,11 +50,16 @@ template class MegaHonkTests : public ::testing::Test { } /** - * @brief Construct and a verify a Honk proof + * @brief Construct and a verify a Honk proof using a specified structured trace * */ bool construct_and_verify_honk_proof_with_structured_trace(auto& builder, TraceSettings& trace_settings) { + // no ZK flavor for now + using Prover = UltraProver_; + using Verifier = UltraVerifier_; + using VerificationKey = typename MegaFlavor::VerificationKey; + using DeciderProvingKey = DeciderProvingKey_; auto proving_key = std::make_shared(builder, trace_settings); Prover prover(proving_key); From 21dc52eee0b80a47332c98f6ad47fc083b51af5c Mon Sep 17 00:00:00 2001 From: iakovenkos Date: Fri, 8 Nov 2024 19:16:42 +0000 Subject: [PATCH 24/25] fix StructuredTraceOverflow test for MegaZKFlavor --- barretenberg/cpp/src/barretenberg/flavor/flavor.hpp | 2 +- barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp index 3b7fa66b826..b59338873cb 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp @@ -360,7 +360,7 @@ template concept IsUltraPlonkFlavor = IsAnyOf; template -concept IsUltraPlonkOrHonk = IsAnyOf; +concept IsUltraPlonkOrHonk = IsAnyOf; template concept IsHonkFlavor = IsAnyOf; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp index 188b6bde3b4..92159f626a6 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp @@ -255,7 +255,7 @@ TYPED_TEST(MegaHonkTests, MultipleCircuitsHonkAndMerge) */ TYPED_TEST(MegaHonkTests, StructuredTraceOverflow) { - using Flavor = MegaFlavor; + using Flavor = TypeParam; using Builder = Flavor::CircuitBuilder; TraceSettings trace_settings{ TraceStructure::TINY_TEST }; From 94a11b109c90ef5c9658d78caac566638f5ea530 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 11 Nov 2024 14:23:22 +0000 Subject: [PATCH 25/25] cleanup and clarify comments per review --- .../ultra_honk/decider_proving_key.cpp | 17 ++++++++++------- .../ultra_honk/decider_proving_key.hpp | 8 +++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp index eec947a7b39..e89d27d23db 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.cpp @@ -88,10 +88,10 @@ void DeciderProvingKey_::construct_databus_polynomials(Circuit& circuit) * placed into an overflow block which can contain arbitrary gate types. * @note One sublety is that gates at row i may in general utilize the values at row i+1 via shifts. If the last row in * a full-capacity block is such a gate, then moving the overflow out of sequence will cause that gate not to be - * satisfied. To avoid this, when a block overflows, the final gate in the block is duplicated, once as a dummy gate in - * the main block (so that the prior gate can read into it but it does not itself try to read into the next row) and - * again as a normal gate in the overflow block. Therefore, the total number of gates in the circuit increases by one - * for each block that overflows. + * satisfied. To avoid this, when a block overflows, the final gate in the block is duplicated, once in the main block + * with the selectors turned off but the wires values maintained (so that the prior gate can read into it but it does + * not itself try to read into the next row) and again as a normal gate in the overflow block. Therefore, the total + * number of gates in the circuit increases by one for each block that overflows. * * @tparam Flavor * @param circuit @@ -112,8 +112,11 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc size_t block_size = block.size(); uint32_t fixed_block_size = block.get_fixed_size(); if (block_size > fixed_block_size && block != overflow_block) { - // We dont handle this case + // Disallow overflow in blocks that are not expected to be used by App circuits ASSERT(!block.is_pub_inputs); + if constexpr (IsGoblinFlavor) { + ASSERT(block != blocks.ecc_op); + } // Set has_overflow to true if at least one block exceeds its capacity blocks.has_overflow = true; @@ -163,8 +166,8 @@ void DeciderProvingKey_::move_structured_trace_overflow_to_overflow_bloc } selector.resize(fixed_block_size); // shrink the main block to its max capacity } - // Convert final gate in the main block to a 'dummy' gate by turning off all gate selectors. This ensures it - // does not erroneously read into the next row. (An 'active' duplicate exists in the overflow block). + // Convert duplicated final gate in the main block to a 'dummy' gate by turning off all selectors. This + // ensures it can be read into by the previous gate but does not itself try to read into the next gate. for (auto& selector : block.get_gate_selectors()) { selector.back() = 0; } diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp index 41eac9a4af9..eebf9cc6439 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp @@ -86,13 +86,11 @@ template class DeciderProvingKey_ { vinfo("constructing proving key"); proving_key = ProvingKey(dyadic_circuit_size, circuit.public_inputs.size(), commitment_key); - if (IsGoblinFlavor && !is_structured) { + // If not using structured trace OR if using structured trace but overflow has occurred (overflow block in + // use), allocate full size polys + if ((IsGoblinFlavor && !is_structured) || (is_structured && circuit.blocks.has_overflow)) { // Allocate full size polynomials proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); - } - // If using structured trace but overflow has occurred (overflow block in use), allocate full size polys - else if (is_structured && circuit.blocks.has_overflow) { - proving_key.polynomials = typename Flavor::ProverPolynomials(dyadic_circuit_size); } else { // Allocate only a correct amount of memory for each polynomial // Allocate the wires and selectors polynomials {