From 7d5e0563edf3c7397ca994033b703149242cc24c Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:29:51 +0000 Subject: [PATCH 1/3] feat: Prototype native merkle trees (#4457) This PR contains work around understanding the performance of our hash functions and merkle trees. 1. Parameterise our existing MerkleTree and NulliferTree to accept any compatible hash function 2. Creates a new native AppendOnlyTree only allowing addition of leaves at the end 3. Creates a new multi-threaded native IndexedMerkleTree. 4. Add tests to validate new trees against previous ones 5. Adds benchmarks for hash functions and new trees. 6. Exposes poseidon hash functions over the WASM and benchmarks it. ### Benchmarks **Hash Functions** **Pedersen - Native** ``` ------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations ------------------------------------------------------------------------------------------ native_pedersen_hash_pair_bench/min_time:3.000 0.178 ms 0.178 ms 23450 ``` **Pedersen - Single hash over WASM 1000 times** Executed 1000 hashes at an average 593.7273874282837us / hash **Pedersen - 1024 hashes over WASM 10 times** Executed 10240 hashes at an average 581.7138370126486us / hash **Poseidon - Native** ``` ---------------------------------------------------------------------------------- Benchmark Time CPU Iterations ---------------------------------------------------------------------------------- poseiden_hash_bench 0.012 ms 0.012 ms 57022 ``` **Poseidon - Single hash over WASM 1000 times** Executed 1000 hashes at an average 48.4412202835083us / hash **Poseidon - 1024 hashes over WASM 10 times** Executed 10240 hashes at an average 37.44762847200036us / hash ### Merkle Trees The following benchmarks follow the format of {benchmark-type}/{batch-insertion-size}/{iteration-count}. **Append Only Tree** ``` ------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations ------------------------------------------------------------------------------------------------ append_only_tree_bench/2/iterations:100 5.75 ms 5.75 ms 100 append_only_tree_bench/4/iterations:100 6.02 ms 6.02 ms 100 append_only_tree_bench/8/iterations:100 6.65 ms 6.65 ms 100 append_only_tree_bench/16/iterations:100 7.89 ms 7.89 ms 100 append_only_tree_bench/32/iterations:100 10.7 ms 10.7 ms 100 append_only_tree_bench/64/iterations:100 16.3 ms 16.3 ms 100 append_only_tree_bench/128/iterations:100 27.8 ms 27.8 ms 100 append_only_tree_bench/2/iterations:1000 0.395 ms 0.395 ms 1000 append_only_tree_bench/4/iterations:1000 0.407 ms 0.408 ms 1000 append_only_tree_bench/8/iterations:1000 0.454 ms 0.454 ms 1000 append_only_tree_bench/16/iterations:1000 0.531 ms 0.531 ms 1000 append_only_tree_bench/32/iterations:1000 0.726 ms 0.726 ms 1000 append_only_tree_bench/64/iterations:1000 1.11 ms 1.11 ms 1000 append_only_tree_bench/128/iterations:1000 1.89 ms 1.89 ms 1000 ``` **Indexed Tree** ``` ---------------------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations ---------------------------------------------------------------------------------------------------------- single_thread_indexed_tree_bench/2/iterations:100 18.2 ms 18.2 ms 100 single_thread_indexed_tree_bench/4/iterations:100 31.3 ms 31.3 ms 100 single_thread_indexed_tree_bench/8/iterations:100 57.2 ms 57.2 ms 100 single_thread_indexed_tree_bench/16/iterations:100 108 ms 108 ms 100 single_thread_indexed_tree_bench/32/iterations:100 210 ms 210 ms 100 single_thread_indexed_tree_bench/64/iterations:100 415 ms 415 ms 100 single_thread_indexed_tree_bench/128/iterations:100 824 ms 824 ms 100 multi_thread_indexed_tree_bench/2/iterations:100 12.8 ms 6.65 ms 100 multi_thread_indexed_tree_bench/4/iterations:100 14.8 ms 7.31 ms 100 multi_thread_indexed_tree_bench/8/iterations:100 18.5 ms 8.80 ms 100 multi_thread_indexed_tree_bench/16/iterations:100 23.9 ms 12.0 ms 100 multi_thread_indexed_tree_bench/32/iterations:100 35.1 ms 18.5 ms 100 multi_thread_indexed_tree_bench/64/iterations:100 59.3 ms 31.6 ms 100 multi_thread_indexed_tree_bench/128/iterations:100 109 ms 66.5 ms 100 single_thread_indexed_tree_bench/2/iterations:1000 1.26 ms 1.26 ms 1000 single_thread_indexed_tree_bench/4/iterations:1000 2.15 ms 2.15 ms 1000 single_thread_indexed_tree_bench/8/iterations:1000 3.88 ms 3.88 ms 1000 single_thread_indexed_tree_bench/16/iterations:1000 7.41 ms 7.41 ms 1000 single_thread_indexed_tree_bench/32/iterations:1000 14.6 ms 14.6 ms 1000 single_thread_indexed_tree_bench/64/iterations:1000 28.7 ms 28.7 ms 1000 single_thread_indexed_tree_bench/128/iterations:1000 56.8 ms 56.8 ms 1000 multi_thread_indexed_tree_bench/2/iterations:1000 1.01 ms 0.812 ms 1000 multi_thread_indexed_tree_bench/4/iterations:1000 1.17 ms 0.865 ms 1000 multi_thread_indexed_tree_bench/8/iterations:1000 1.46 ms 0.979 ms 1000 multi_thread_indexed_tree_bench/16/iterations:1000 2.05 ms 1.23 ms 1000 multi_thread_indexed_tree_bench/32/iterations:1000 3.24 ms 1.70 ms 1000 multi_thread_indexed_tree_bench/64/iterations:1000 5.84 ms 2.54 ms 1000 multi_thread_indexed_tree_bench/128/iterations:1000 13.3 ms 6.36 ms 1000 ``` --------- Co-authored-by: ludamad --- barretenberg/cpp/scripts/stdlib-tests | 1 - barretenberg/cpp/src/CMakeLists.txt | 2 +- .../src/barretenberg/benchmark/CMakeLists.txt | 6 +- .../append_only_tree_bench/CMakeLists.txt | 1 + .../append_only_tree.bench.cpp | 54 +++ .../benchmark/goblin_bench/CMakeLists.txt | 2 +- .../indexed_tree_bench/CMakeLists.txt | 1 + .../indexed_tree_bench/indexed_tree.bench.cpp | 87 ++++ .../benchmark/ivc_bench/CMakeLists.txt | 2 +- .../merkle_tree_bench/CMakeLists.txt | 1 + .../merkle_tree_bench}/merkle_tree.bench.cpp | 20 +- .../benchmark/poseidon2_bench/CMakeLists.txt | 1 + .../poseidon2_bench}/poseidon2.bench.cpp | 18 +- .../benchmark/ultra_bench/CMakeLists.txt | 2 +- .../benchmark/ultra_bench/mock_proofs.hpp | 8 +- .../src/barretenberg/crypto/CMakeLists.txt | 3 +- .../crypto/merkle_tree/CMakeLists.txt | 1 + .../append_only_tree/append_only_tree.hpp | 195 +++++++++ .../append_only_tree.test.cpp | 127 ++++++ .../crypto/merkle_tree/array_store.hpp | 37 ++ .../{stdlib => crypto}/merkle_tree/hash.hpp | 24 +- .../merkle_tree/hash.test.cpp | 18 +- .../merkle_tree/hash_path.hpp | 16 +- .../{stdlib => crypto}/merkle_tree/index.hpp | 0 .../merkle_tree/indexed_tree/indexed_leaf.hpp | 25 ++ .../merkle_tree/indexed_tree/indexed_tree.hpp | 379 ++++++++++++++++++ .../indexed_tree/indexed_tree.test.cpp | 331 +++++++++++++++ .../merkle_tree/indexed_tree/leaves_cache.cpp | 49 +++ .../merkle_tree/indexed_tree/leaves_cache.hpp | 27 ++ .../merkle_tree/membership.hpp | 31 +- .../merkle_tree/membership.test.cpp | 41 +- .../merkle_tree/memory_store.hpp | 4 +- .../crypto/merkle_tree/memory_tree.hpp | 114 ++++++ .../merkle_tree/memory_tree.test.cpp | 24 +- .../merkle_tree/merkle_tree.hpp} | 164 ++++++-- .../merkle_tree/merkle_tree.test.cpp | 36 +- .../nullifier_tree/nullifier_leaf.hpp | 23 +- .../nullifier_tree/nullifier_memory_tree.hpp | 174 ++++++++ .../nullifier_memory_tree.test.cpp | 161 ++++---- .../nullifier_tree/nullifier_tree.cpp | 57 +-- .../nullifier_tree/nullifier_tree.hpp | 37 ++ .../nullifier_tree/nullifier_tree.test.cpp | 25 +- .../crypto/pedersen_hash/c_bind.cpp | 29 +- .../crypto/pedersen_hash/c_bind.hpp | 8 +- .../barretenberg/crypto/poseidon2/c_bind.cpp | 32 ++ .../barretenberg/crypto/poseidon2/c_bind.hpp | 10 + .../cpp/src/barretenberg/dsl/CMakeLists.txt | 2 +- .../cpp/src/barretenberg/dsl/types.hpp | 4 +- .../src/barretenberg/goblin/CMakeLists.txt | 2 +- .../src/barretenberg/goblin/mock_circuits.hpp | 6 +- .../proofs/join_split/CMakeLists.txt | 2 +- .../join_split/compute_circuit_data.cpp | 4 +- .../proofs/join_split/join_split.cpp | 2 +- .../proofs/join_split/join_split.test.cpp | 10 +- .../proofs/join_split/join_split_circuit.cpp | 12 +- .../join_split/join_split_js_parity.test.cpp | 8 +- .../proofs/join_split/join_split_tx.hpp | 6 +- .../proofs/notes/CMakeLists.txt | 2 +- .../barretenberg/join_split_example/types.hpp | 4 +- .../src/barretenberg/stdlib/CMakeLists.txt | 3 +- .../stdlib/hash/pedersen/pedersen.bench.cpp | 12 + .../stdlib/merkle_tree/CMakeLists.txt | 1 - .../indexed_tree/indexed_tree.test.cpp | 331 +++++++++++++++ .../stdlib/merkle_tree/memory_tree.cpp | 77 ---- .../stdlib/merkle_tree/memory_tree.hpp | 44 -- .../stdlib/merkle_tree/merkle_tree.hpp | 98 ----- .../nullifier_tree/nullifier_memory_tree.cpp | 73 ---- .../nullifier_tree/nullifier_memory_tree.hpp | 92 ----- .../nullifier_tree/nullifier_tree.hpp | 39 -- .../src/barretenberg/stdlib/types/ultra.hpp | 7 +- barretenberg/exports.json | 52 +++ barretenberg/scripts/bindgen.sh | 2 +- barretenberg/scripts/c_bind_files.txt | 1 + .../__snapshots__/poseidon.test.ts.snap | 40 ++ .../ts/src/barretenberg/pedersen.test.ts | 23 ++ .../ts/src/barretenberg/poseidon.test.ts | 39 ++ barretenberg/ts/src/barretenberg_api/index.ts | 72 ++++ 77 files changed, 2743 insertions(+), 735 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt rename barretenberg/cpp/src/barretenberg/{stdlib/merkle_tree => benchmark/merkle_tree_bench}/merkle_tree.bench.cpp (77%) create mode 100644 barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt rename barretenberg/cpp/src/barretenberg/{crypto/poseidon2 => benchmark/poseidon2_bench}/poseidon2.bench.cpp (57%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/hash.hpp (73%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/hash.test.cpp (75%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/hash_path.hpp (83%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/index.hpp (100%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/membership.hpp (92%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/membership.test.cpp (89%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/memory_store.hpp (96%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/memory_tree.test.cpp (74%) rename barretenberg/cpp/src/barretenberg/{stdlib/merkle_tree/merkle_tree.cpp => crypto/merkle_tree/merkle_tree.hpp} (67%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/merkle_tree.test.cpp (78%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/nullifier_tree/nullifier_leaf.hpp (76%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp (67%) rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/nullifier_tree/nullifier_tree.cpp (51%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp rename barretenberg/cpp/src/barretenberg/{stdlib => crypto}/merkle_tree/nullifier_tree/nullifier_tree.test.cpp (81%) create mode 100644 barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp create mode 100644 barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp create mode 100644 barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap create mode 100644 barretenberg/ts/src/barretenberg/poseidon.test.ts diff --git a/barretenberg/cpp/scripts/stdlib-tests b/barretenberg/cpp/scripts/stdlib-tests index 6cb946112947..75285f483d64 100644 --- a/barretenberg/cpp/scripts/stdlib-tests +++ b/barretenberg/cpp/scripts/stdlib-tests @@ -3,7 +3,6 @@ stdlib_aes128_tests stdlib_blake2s_tests stdlib_blake3s_tests stdlib_ecdsa_tests -stdlib_merkle_tree_tests stdlib_pedersen_commitment_tests stdlib_pedersen_hash_tests stdlib_poseidon2_tests diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 3c996c513dcc..3eea79dfdbf8 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -127,7 +127,7 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ + $ $ $ $ diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index ae6fa95da27e..145919222651 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -7,5 +7,9 @@ add_subdirectory(pippenger_bench) add_subdirectory(plonk_bench) add_subdirectory(protogalaxy_bench) add_subdirectory(relations_bench) +add_subdirectory(widgets_bench) +add_subdirectory(poseidon2_bench) +add_subdirectory(merkle_tree_bench) +add_subdirectory(indexed_tree_bench) +add_subdirectory(append_only_tree_bench) add_subdirectory(ultra_bench) -add_subdirectory(widgets_bench) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..7270e7e95305 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp new file mode 100644 index 000000000000..bde03dcb4ff9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -0,0 +1,54 @@ +#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include + +using namespace benchmark; +using namespace bb::crypto::merkle_tree; + +using Pedersen = AppendOnlyTree; +using Poseidon2 = AppendOnlyTree; + +const size_t TREE_DEPTH = 32; +const size_t MAX_BATCH_SIZE = 128; + +namespace { +auto& random_engine = bb::numeric::get_randomness(); +} // namespace + +template void perform_batch_insert(TreeType& tree, const std::vector& values) +{ + tree.add_values(values); +} + +template void append_only_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values); + } +} +BENCHMARK(append_only_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); +BENCHMARK(append_only_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt index 41d407c044dd..2dfa66906e97 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives) +barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..998bbdf55def --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp new file mode 100644 index 000000000000..5e7c0f10abb6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -0,0 +1,87 @@ +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include + +using namespace benchmark; +using namespace bb::crypto::merkle_tree; + +using Poseidon2 = IndexedTree; +using Pedersen = IndexedTree; + +const size_t TREE_DEPTH = 32; +const size_t MAX_BATCH_SIZE = 128; + +namespace { +auto& random_engine = bb::numeric::get_randomness(); +} // namespace + +template +void perform_batch_insert(TreeType& tree, const std::vector& values, bool single_threaded) +{ + tree.add_or_update_values(values, single_threaded); +} + +template void multi_thread_indexed_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth, batch_size); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values, false); + } +} + +template void single_thread_indexed_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth, batch_size); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values, true); + } +} +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); + +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt index 3f0c92458505..6f622044421a 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives) +barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..f497e924d56a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp similarity index 77% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp rename to barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp index 49871e881e6f..cdc18ed6c801 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp @@ -1,15 +1,17 @@ -#include "merkle_tree.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" #include "barretenberg/numeric/random/engine.hpp" -#include "hash.hpp" -#include "memory_store.hpp" #include using namespace benchmark; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using TreeType = MerkleTree; namespace { auto& engine = bb::numeric::get_debug_randomness(); -} +} // namespace constexpr size_t DEPTH = 256; constexpr size_t MAX = 4096; @@ -33,7 +35,7 @@ BENCHMARK(hash)->MinTime(5); void update_first_element(State& state) noexcept { MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); for (auto _ : state) { db.update_element(0, VALUES[1]); @@ -46,7 +48,7 @@ void update_elements(State& state) noexcept for (auto _ : state) { state.PauseTiming(); MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); state.ResumeTiming(); for (size_t i = 0; i < (size_t)state.range(0); ++i) { db.update_element(i, VALUES[i]); @@ -60,10 +62,10 @@ void update_random_elements(State& state) noexcept for (auto _ : state) { state.PauseTiming(); MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); for (size_t i = 0; i < (size_t)state.range(0); i++) { state.PauseTiming(); - auto index = MerkleTree::index_t(engine.get_random_uint256()); + auto index = TreeType::index_t(engine.get_random_uint256()); state.ResumeTiming(); db.update_element(index, VALUES[i]); } diff --git a/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt new file mode 100644 index 000000000000..87e6e3d63dd6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(poseidon2_bench crypto_poseidon2) diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp similarity index 57% rename from barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp rename to barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp index 6673734acdc4..4118e4925601 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp @@ -1,4 +1,4 @@ -#include "./poseidon2.hpp" +#include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include @@ -24,4 +24,20 @@ void native_poseidon2_commitment_bench(State& state) noexcept } BENCHMARK(native_poseidon2_commitment_bench)->Arg(10)->Arg(1000)->Arg(10000); +grumpkin::fq poseiden_hash_impl(const grumpkin::fq& x, const grumpkin::fq& y) +{ + std::vector to_hash{ x, y }; + return bb::crypto::Poseidon2::hash(to_hash); +} + +void poseiden_hash_bench(State& state) noexcept +{ + grumpkin::fq x = grumpkin::fq::random_element(); + grumpkin::fq y = grumpkin::fq::random_element(); + for (auto _ : state) { + DoNotOptimize(poseiden_hash_impl(x, y)); + } +} +BENCHMARK(poseiden_hash_bench)->Unit(benchmark::kMillisecond); + BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt index 323ad1613b0d..bd67d08ae716 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt @@ -2,5 +2,5 @@ barretenberg_module(ultra_bench ultra_honk stdlib_sha256 stdlib_keccak - stdlib_merkle_tree + crypto_merkle_tree stdlib_recursion) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp index da9c1f2f1084..05942e6afe35 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp @@ -2,6 +2,10 @@ #include #include +#include "barretenberg/crypto/merkle_tree/membership.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/memory_tree.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/goblin/mock_circuits.hpp" #include "barretenberg/plonk/composer/standard_composer.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" @@ -9,10 +13,6 @@ #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" #include "barretenberg/stdlib/hash/keccak/keccak.hpp" #include "barretenberg/stdlib/hash/sha256/sha256.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/bool/bool.hpp" #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt index 6efc1f824070..5ea623e144e8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt @@ -9,4 +9,5 @@ add_subdirectory(schnorr) add_subdirectory(sha256) add_subdirectory(ecdsa) add_subdirectory(aes128) -add_subdirectory(poseidon2) \ No newline at end of file +add_subdirectory(poseidon2) +add_subdirectory(merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt new file mode 100644 index 000000000000..2c48323594be --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(crypto_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp new file mode 100644 index 000000000000..7170a9292ea7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -0,0 +1,195 @@ +#pragma once +#include "../hash_path.hpp" + +namespace bb::crypto::merkle_tree { + +using namespace bb; + +typedef uint256_t index_t; + +/** + * @brief Implements a simple append-only merkle tree + * Accepts template argument of the type of store backing the tree and the hashing policy + * + */ +template class AppendOnlyTree { + public: + AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id = 0); + AppendOnlyTree(AppendOnlyTree const& other) = delete; + AppendOnlyTree(AppendOnlyTree&& other) = delete; + virtual ~AppendOnlyTree(); + + /** + * @brief Adds a single value to the end of the tree + */ + virtual fr add_value(const fr& value); + + /** + * @brief Adds the given set of values to the end of the tree + */ + virtual fr add_values(const std::vector& values); + + /** + * @brief Returns the index of the right-most populated leaf in the tree + */ + index_t size() const; + + /** + * @brief Returns the root of the tree + */ + fr root() const; + + /** + * @brief Returns the depth of the tree + */ + size_t depth() const; + + /** + * @brief Returns the hash path from the leaf at the given index to the root + */ + fr_hash_path get_hash_path(const index_t& index) const; + + protected: + fr get_element_or_zero(size_t level, const index_t& index) const; + + void write_node(size_t level, const index_t& index, const fr& value); + std::pair read_node(size_t level, const index_t& index) const; + + Store& store_; + size_t depth_; + uint8_t tree_id_; + std::vector zero_hashes_; + fr root_; + index_t size_; +}; + +template +AppendOnlyTree::AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id) + : store_(store) + , depth_(depth) + , tree_id_(tree_id) +{ + zero_hashes_.resize(depth + 1); + + // Create the zero hashes for the tree + auto current = HashingPolicy::zero_hash(); + for (size_t i = depth; i > 0; --i) { + zero_hashes_[i] = current; + current = HashingPolicy::hash_pair(current, current); + } + zero_hashes_[0] = current; + root_ = current; +} + +template AppendOnlyTree::~AppendOnlyTree() {} + +template index_t AppendOnlyTree::size() const +{ + return size_; +} + +template fr AppendOnlyTree::root() const +{ + return root_; +} + +template size_t AppendOnlyTree::depth() const +{ + return depth_; +} + +template +fr_hash_path AppendOnlyTree::get_hash_path(const index_t& index) const +{ + fr_hash_path path; + index_t current_index = index; + + for (size_t level = depth_; level > 0; --level) { + bool is_right = bool(current_index & 0x01); + fr right_value = + is_right ? get_element_or_zero(level, current_index) : get_element_or_zero(level, current_index + 1); + fr left_value = + is_right ? get_element_or_zero(level, current_index - 1) : get_element_or_zero(level, current_index); + path.push_back(std::make_pair(left_value, right_value)); + current_index >>= 1; + } + return path; +} + +template fr AppendOnlyTree::add_value(const fr& value) +{ + return add_values(std::vector{ value }); +} + +template +fr AppendOnlyTree::add_values(const std::vector& values) +{ + index_t index = size(); + size_t number_to_insert = values.size(); + size_t level = depth_; + std::vector hashes = values; + + // Add the values at the leaf nodes of the tree + for (size_t i = 0; i < number_to_insert; ++i) { + write_node(level, index + i, hashes[i]); + } + + // Hash the values as a sub tree and insert them + while (number_to_insert > 1) { + number_to_insert >>= 1; + index >>= 1; + --level; + for (size_t i = 0; i < number_to_insert; ++i) { + hashes[i] = HashingPolicy::hash_pair(hashes[i * 2], hashes[i * 2 + 1]); + write_node(level, index + i, hashes[i]); + } + } + + // Hash from the root of the sub-tree to the root of the overall tree + fr new_hash = hashes[0]; + while (level > 0) { + bool is_right = bool(index & 0x01); + fr left_hash = is_right ? get_element_or_zero(level, index - 1) : new_hash; + fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1); + new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + index >>= 1; + --level; + write_node(level, index, new_hash); + } + size_ += values.size(); + root_ = new_hash; + return root_; +} + +template +fr AppendOnlyTree::get_element_or_zero(size_t level, const index_t& index) const +{ + const std::pair read_data = read_node(level, index); + if (read_data.first) { + return read_data.second; + } + ASSERT(level > 0 && level < zero_hashes_.size()); + return zero_hashes_[level]; +} + +template +void AppendOnlyTree::write_node(size_t level, const index_t& index, const fr& value) +{ + std::vector buf; + write(buf, value); + store_.put(level, size_t(index), buf); +} + +template +std::pair AppendOnlyTree::read_node(size_t level, const index_t& index) const +{ + std::vector buf; + bool available = store_.get(level, size_t(index), buf); + if (!available) { + return std::make_pair(false, fr::zero()); + } + fr value = from_buffer(buf, 0); + return std::make_pair(true, value); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp new file mode 100644 index 000000000000..e25f53ba1ca8 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -0,0 +1,127 @@ +#include "append_only_tree.hpp" +#include "../array_store.hpp" +#include "../memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +inline void print_tree(const size_t depth, std::vector hashes, std::string const& msg) +{ + info("\n", msg); + size_t offset = 0; + for (size_t i = 0; i < depth; i++) { + info("i = ", i); + size_t layer_size = (1UL << (depth - i)); + for (size_t j = 0; j < layer_size; j++) { + info("j = ", j, ": ", hashes[offset + j]); + } + offset += layer_size; + } +} + +TEST(stdlib_append_only_tree, can_create) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); +} + +TEST(stdlib_append_only_tree, can_add_value) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); + + memdb.update_element(0, VALUES[0]); + tree.add_value(VALUES[0]); + + EXPECT_EQ(tree.root(), memdb.root()); + EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); +} + +TEST(stdlib_append_only_tree, test_size) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + + EXPECT_EQ(tree.size(), 0ULL); + + // Add a new non-zero leaf at index 0. + tree.add_value(30); + EXPECT_EQ(tree.size(), 1ULL); + + // Add second. + tree.add_value(10); + EXPECT_EQ(tree.size(), 2ULL); + + // Add third. + tree.add_value(20); + EXPECT_EQ(tree.size(), 3ULL); + + // Add forth but with same value. + tree.add_value(20); + EXPECT_EQ(tree.size(), 4ULL); +} + +TEST(stdlib_append_only_tree, can_add_multiple_values) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + for (size_t i = 0; i < NUM_VALUES; ++i) { + fr mock_root = memdb.update_element(i, VALUES[i]); + fr tree_root = tree.add_value(VALUES[i]); + EXPECT_EQ(mock_root, tree_root); + + EXPECT_EQ(memdb.get_hash_path(0), tree.get_hash_path(0)); + EXPECT_EQ(memdb.get_hash_path(i), tree.get_hash_path(i)); + } +} + +TEST(stdlib_append_only_tree, can_be_filled) +{ + constexpr size_t depth = 3; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); + + for (size_t i = 0; i < 8; i++) { + memdb.update_element(i, VALUES[i]); + tree.add_value(VALUES[i]); + } + + EXPECT_EQ(tree.root(), memdb.root()); + EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); + EXPECT_EQ(tree.get_hash_path(7), memdb.get_hash_path(7)); +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp new file mode 100644 index 000000000000..ade8472b1b91 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp @@ -0,0 +1,37 @@ +#pragma once +#include "barretenberg/stdlib/primitives/field/field.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * @brief A very basic 2-d array for use as a backing store for merkle trees. + * Can store up to 'indices' nodes per row and 'levels' rows. + */ +class ArrayStore { + + public: + ArrayStore(size_t levels, size_t indices = 1024) + : map_(std::vector>>>( + levels + 1, + std::vector>>( + indices, std::pair>(false, std::vector())))) + {} + ~ArrayStore() {} + + void put(size_t level, size_t index, const std::vector& data) + { + map_[level][index] = std::make_pair(true, data); + } + bool get(size_t level, size_t index, std::vector& data) const + { + const std::pair>& slot = map_[level][index]; + if (slot.first) { + data = slot.second; + } + return slot.first; + } + + private: + std::vector>>> map_; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp similarity index 73% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp index 4e5188206804..79a0b7e09c36 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp @@ -3,12 +3,32 @@ #include "barretenberg/crypto/blake2s/blake2s.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/crypto/pedersen_hash/pedersen.hpp" +#include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/stdlib/hash/blake2s/blake2s.hpp" #include "barretenberg/stdlib/hash/pedersen/pedersen.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { + +struct PedersenHashPolicy { + static fr hash(const std::vector& inputs) { return crypto::pedersen_hash::hash(inputs); } + + static fr hash_pair(const fr& lhs, const fr& rhs) { return hash(std::vector({ lhs, rhs })); } + + static fr zero_hash() { return fr::zero(); } +}; + +struct Poseidon2HashPolicy { + static fr hash(const std::vector& inputs) + { + return bb::crypto::Poseidon2::hash(inputs); + } + + static fr hash_pair(const fr& lhs, const fr& rhs) { return hash(std::vector({ lhs, rhs })); } + + static fr zero_hash() { return fr::zero(); } +}; inline bb::fr hash_pair_native(bb::fr const& lhs, bb::fr const& rhs) { @@ -63,4 +83,4 @@ inline std::vector compute_tree_native(std::vector const& input) return tree; } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp similarity index 75% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp index 2e420fb15383..a8eeec7cc697 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp @@ -2,29 +2,29 @@ #include "memory_tree.hpp" #include +#include "barretenberg/crypto/merkle_tree/membership.hpp" #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" using namespace bb; -using namespace bb::stdlib; +using namespace bb::crypto; using Builder = UltraCircuitBuilder; -using field_ct = field_t; -using witness_ct = witness_t; +using field_ct = bb::stdlib::field_t; +using witness_ct = bb::stdlib::witness_t; -TEST(stdlib_merkle_tree_hash, hash_native_vs_circuit) +TEST(crypto_merkle_tree_hash, hash_native_vs_circuit) { fr x = uint256_t(0x5ec473eb273a8011, 0x50160109385471ca, 0x2f3095267e02607d, 0x02586f4a39e69b86); Builder builder = Builder(); witness_ct y = witness_ct(&builder, x); - field_ct z = pedersen_hash::hash({ y, y }); + field_ct z = bb::stdlib::pedersen_hash::hash({ y, y }); auto zz = merkle_tree::hash_pair_native(x, x); EXPECT_EQ(z.get_value(), zz); } -TEST(stdlib_merkle_tree_hash, compute_tree_root_native_vs_circuit) +TEST(crypto_merkle_tree_hash, compute_tree_root_native_vs_circuit) { Builder builder = Builder(); std::vector inputs; @@ -42,10 +42,10 @@ TEST(stdlib_merkle_tree_hash, compute_tree_root_native_vs_circuit) EXPECT_EQ(z.get_value(), zz); } -TEST(stdlib_merkle_tree_hash, compute_tree_native) +TEST(crypto_merkle_tree_hash, compute_tree_native) { constexpr size_t depth = 2; - merkle_tree::MemoryTree mem_tree(depth); + merkle_tree::MemoryTree mem_tree(depth); std::vector leaves; for (size_t i = 0; i < (size_t(1) << depth); i++) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp similarity index 83% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp index f661b290c488..fb9130bda2e2 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp @@ -4,13 +4,12 @@ #include #include -namespace bb::stdlib::merkle_tree { - -using namespace bb; +namespace bb::crypto::merkle_tree { +using fr = bb::stdlib::fr; using fr_hash_path = std::vector>; using fr_sibling_path = std::vector; -template using hash_path = std::vector, field_t>>; +template using hash_path = std::vector, bb::stdlib::field_t>>; inline fr_hash_path get_new_hash_path(fr_hash_path const& old_path, uint128_t index, fr const& value) { @@ -42,7 +41,8 @@ template inline hash_path create_witness_hash_path(Ctx& ctx, { hash_path result; std::transform(input.begin(), input.end(), std::back_inserter(result), [&](auto const& v) { - return std::make_pair(field_t(witness_t(&ctx, v.first)), field_t(witness_t(&ctx, v.second))); + return std::make_pair(bb::stdlib::field_t(bb::stdlib::witness_t(&ctx, v.first)), + bb::stdlib::field_t(bb::stdlib::witness_t(&ctx, v.second))); }); return result; } @@ -61,13 +61,13 @@ inline fr zero_hash_at_height(size_t height) return current; } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree // We add to std namespace as fr_hash_path is actually a std::vector, and this is the only way // to achieve effective ADL. namespace std { template -inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::hash_path const& path) +inline std::ostream& operator<<(std::ostream& os, bb::crypto::merkle_tree::hash_path const& path) { os << "[\n"; for (size_t i = 0; i < path.size(); ++i) { @@ -77,7 +77,7 @@ inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::hash_ return os; } -inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::fr_hash_path const& path) +inline std::ostream& operator<<(std::ostream& os, bb::crypto::merkle_tree::fr_hash_path const& path) { os << "[\n"; for (size_t i = 0; i < path.size(); ++i) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/index.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/index.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/index.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/index.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp new file mode 100644 index 000000000000..bd5e9859cbec --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "barretenberg/stdlib/primitives/field/field.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +struct indexed_leaf { + fr value; + index_t nextIndex; + fr nextValue; + + bool operator==(indexed_leaf const&) const = default; + + std::ostream& operator<<(std::ostream& os) + { + os << "value = " << value << "\nnextIdx = " << nextIndex << "\nnextVal = " << nextValue; + return os; + } + + std::vector get_hash_inputs() const { return std::vector({ value, nextIndex, nextValue }); } +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp new file mode 100644 index 000000000000..3afb64e91634 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -0,0 +1,379 @@ +#pragma once +#include "../../../common/thread.hpp" +#include "../append_only_tree/append_only_tree.hpp" +#include "../hash.hpp" +#include "../hash_path.hpp" +#include "indexed_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +/** + * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up + * the level of the tree. + * + */ +class LevelSignal { + public: + LevelSignal(size_t initial_level) + : signal_(initial_level){}; + ~LevelSignal(){}; + LevelSignal(const LevelSignal& other) + : signal_(other.signal_.load()) + {} + LevelSignal(const LevelSignal&& other) = delete; + + /** + * @brief Causes the thread to wait until the required level has been signalled + * @param level The required level + * + */ + void wait_for_level(size_t level) + { + size_t current_level = signal_.load(); + while (current_level > level) { + signal_.wait(current_level); + current_level = signal_.load(); + } + } + + /** + * @brief Signals that the given level has been passed + * @param level The level to be signalled + * + */ + void signal_level(size_t level) + { + signal_.store(level); + signal_.notify_all(); + } + + private: + std::atomic signal_; +}; + +/** + * @brief Implements a parallelised batch insertion indexed tree + * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the + * hashing policy + * + */ +template +class IndexedTree : public AppendOnlyTree { + public: + IndexedTree(Store& store, size_t depth, size_t initial_size = 1, uint8_t tree_id = 0); + IndexedTree(IndexedTree const& other) = delete; + IndexedTree(IndexedTree&& other) = delete; + ~IndexedTree(); + + /** + * @brief Adds or updates a single values in the tree (updates not currently supported) + * @param value The value to be added or updated + * @returns The 'previous' hash paths of all updated values + */ + fr_hash_path add_or_update_value(const fr& value); + + /** + * @brief Adds or updates the given set of values in the tree (updates not currently supported) + * @param values The values to be added or updated + * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking + * @returns The 'previous' hash paths of all updated values + */ + std::vector add_or_update_values(const std::vector& values, bool no_multithreading = false); + + /** + * @brief Adds or updates a single value without returning the previous hash path + * @param value The value to be added or updated + * @returns The new root of the tree + */ + fr add_value(const fr& value) override; + + /** + * @brief Adds or updates the given set of values without returning the previous hash paths + * @param values The values to be added or updated + * @returns The new root of the tree + */ + fr add_values(const std::vector& values) override; + + indexed_leaf get_leaf(const index_t& index); + + using AppendOnlyTree::get_hash_path; + using AppendOnlyTree::root; + using AppendOnlyTree::depth; + + private: + fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); + fr update_leaf_and_hash_to_root(const index_t& index, + const indexed_leaf& leaf, + LevelSignal& leader, + LevelSignal& follower, + fr_hash_path& previous_hash_path); + fr append_subtree(const index_t& start_index); + + using AppendOnlyTree::get_element_or_zero; + using AppendOnlyTree::write_node; + using AppendOnlyTree::read_node; + + private: + using AppendOnlyTree::store_; + using AppendOnlyTree::zero_hashes_; + using AppendOnlyTree::depth_; + using AppendOnlyTree::tree_id_; + using AppendOnlyTree::root_; + LeavesStore leaves_; +}; + +template +IndexedTree::IndexedTree(Store& store, + size_t depth, + size_t initial_size, + uint8_t tree_id) + : AppendOnlyTree(store, depth, tree_id) +{ + ASSERT(initial_size > 0); + zero_hashes_.resize(depth + 1); + + // Create the zero hashes for the tree + indexed_leaf zero_leaf{ 0, 0, 0 }; + auto current = HashingPolicy::hash(zero_leaf.get_hash_inputs()); + for (size_t i = depth; i > 0; --i) { + zero_hashes_[i] = current; + current = HashingPolicy::hash_pair(current, current); + } + zero_hashes_[0] = current; + // Inserts the initial set of leaves as a chain in incrementing value order + for (size_t i = 0; i < initial_size; ++i) { + // Insert the zero leaf to the `leaves` and also to the tree at index 0. + indexed_leaf initial_leaf = indexed_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }; + leaves_.append_leaf(initial_leaf); + } + + // Points the last leaf back to the first + leaves_.set_at_index( + initial_size - 1, + indexed_leaf{ .value = leaves_.get_leaf(initial_size - 1).value, .nextIndex = 0, .nextValue = 0 }, + false); + append_subtree(0); +} + +template +IndexedTree::~IndexedTree() +{} + +template +indexed_leaf IndexedTree::get_leaf(const index_t& index) +{ + return leaves_.get_leaf(index); +} + +template +fr IndexedTree::add_value(const fr& value) +{ + return add_values(std::vector{ value }); +} + +template +fr IndexedTree::add_values(const std::vector& values) +{ + add_or_update_values(values); + return root(); +} + +template +fr_hash_path IndexedTree::add_or_update_value(const fr& value) +{ + return add_or_update_values(std::vector{ value })[0]; +} + +template +std::vector IndexedTree::add_or_update_values( + const std::vector& values, bool no_multithreading) +{ + // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order + struct { + bool operator()(const std::pair& a, const std::pair& b) const + { + return uint256_t(a.first) > uint256_t(b.first); + } + } comp; + std::vector> values_sorted(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + values_sorted[i] = std::make_pair(values[i], i); + } + std::sort(values_sorted.begin(), values_sorted.end(), comp); + + // Now that we have the sorted values we need to identify the leaves that need updating. + // This is performed sequentially and is stored in this 'leaf_insertion' struct + struct leaf_insertion { + index_t low_leaf_index; + indexed_leaf low_leaf; + }; + + std::vector insertions(values.size()); + index_t old_size = leaves_.get_size(); + + for (size_t i = 0; i < values_sorted.size(); ++i) { + fr value = values_sorted[i].first; + index_t index_of_new_leaf = index_t(values_sorted[i].second) + old_size; + + // This gives us the leaf that need updating + index_t current; + bool is_already_present; + std::tie(is_already_present, current) = leaves_.find_low_value(values_sorted[i].first); + indexed_leaf current_leaf = leaves_.get_leaf(current); + + indexed_leaf new_leaf = + indexed_leaf{ .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }; + + // We only handle new values being added. We don't yet handle values being updated + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + current_leaf.nextIndex = index_of_new_leaf; + current_leaf.nextValue = value; + + leaves_.set_at_index(current, current_leaf, false); + leaves_.set_at_index(index_of_new_leaf, new_leaf, true); + } + + // Capture the index and value of the updated 'low' leaf + leaf_insertion& insertion = insertions[i]; + insertion.low_leaf_index = current; + insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + } + + // We now kick off multiple workers to perform the low leaf updates + // We create set of signals to coordinate the workers as the move up the tree + std::vector paths(insertions.size()); + std::vector signals; + // The first signal is set to 0. This ensure the first worker up the tree is not impeded + signals.emplace_back(size_t(0)); + // Workers will follow their leaders up the tree, being trigger by the signal in front of them + for (size_t i = 0; i < insertions.size(); ++i) { + signals.emplace_back(size_t(1 + depth_)); + } + + if (no_multithreading) { + // Execute the jobs in series + for (size_t i = 0; i < insertions.size(); ++i) { + leaf_insertion& insertion = insertions[i]; + update_leaf_and_hash_to_root( + insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); + } + } else { + // Execute the jobs in parallel + parallel_for(insertions.size(), [&](size_t i) { + leaf_insertion& insertion = insertions[i]; + update_leaf_and_hash_to_root( + insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); + }); + } + + // Now that we have updated all of the low leaves, we insert the new leaves as a subtree at the end + root_ = append_subtree(old_size); + + return paths; +} + +template +fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, + const indexed_leaf& leaf) +{ + LevelSignal leader(0); + LevelSignal follower(0); + fr_hash_path hash_path; + return update_leaf_and_hash_to_root(leaf_index, leaf, leader, follower, hash_path); +} + +template +fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, + const indexed_leaf& leaf, + LevelSignal& leader, + LevelSignal& follower, + fr_hash_path& previous_hash_path) +{ + // We are a worker at a specific leaf index. + // We are going to move up the tree and at each node/level: + // 1. Wait for the level above to become 'signalled' as clear for us to write into + // 2. Read the node and it's sibling + // 3. Write the new node value + index_t index = leaf_index; + size_t level = depth_; + fr new_hash = HashingPolicy::hash(leaf.get_hash_inputs()); + + // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about to + // write into) this ensures that our leader is not still reading the leaves + size_t leader_level = depth_ - 1; + leader.wait_for_level(leader_level); + + // Extract the value of the leaf node and it's sibling + bool is_right = bool(index & 0x01); + // extract the current leaf hash values for the previous hash path + fr current_right_value = get_element_or_zero(level, index + (is_right ? 0 : 1)); + fr current_left_value = get_element_or_zero(level, is_right ? (index - 1) : index); + previous_hash_path.push_back(std::make_pair(current_left_value, current_right_value)); + + // Write the new leaf hash in place + write_node(level, index, new_hash); + // Signal that this level has been written + follower.signal_level(level); + + while (level > 0) { + if (level > 1) { + // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we can + // read from it + size_t level_to_read = level - 1; + leader_level = level_to_read; + + leader.wait_for_level(leader_level); + + // Now read the node and it's sibling + index_t index_of_node_above = index >> 1; + bool node_above_is_right = bool(index_of_node_above & 0x01); + fr above_right_value = + get_element_or_zero(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); + fr above_left_value = get_element_or_zero( + level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); + previous_hash_path.push_back(std::make_pair(above_left_value, above_right_value)); + } + + // Now that we have extracted the hash path from the row above + // we can compute the new hash at that level and write it + is_right = bool(index & 0x01); + fr new_right_value = is_right ? new_hash : get_element_or_zero(level, index + 1); + fr new_left_value = is_right ? get_element_or_zero(level, index - 1) : new_hash; + new_hash = HashingPolicy::hash_pair(new_left_value, new_right_value); + index >>= 1; + --level; + if (level > 0) { + // Before we write we need to ensure that our leader has already written to the row above it + // otherwise it could still be reading from this level + leader_level = level - 1; + leader.wait_for_level(leader_level); + } + + // Write this node and signal that it is done + write_node(level, index, new_hash); + follower.signal_level(level); + } + return new_hash; +} + +template +fr IndexedTree::append_subtree(const index_t& start_index) +{ + index_t index = start_index; + size_t number_to_insert = size_t(index_t(leaves_.get_size()) - index); + std::vector hashes_to_append = std::vector(number_to_insert); + + for (size_t i = 0; i < number_to_insert; ++i) { + index_t index_to_insert = index + i; + hashes_to_append[i] = HashingPolicy::hash(leaves_.get_leaf(size_t(index_to_insert)).get_hash_inputs()); + } + + return AppendOnlyTree::add_values(hashes_to_append); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp new file mode 100644 index 000000000000..d1dc13d95e52 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -0,0 +1,331 @@ +#include "indexed_tree.hpp" +#include "../array_store.hpp" +#include "../hash.hpp" +#include "../nullifier_tree/nullifier_memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "leaves_cache.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = Poseidon2HashPolicy; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +TEST(stdlib_indexed_tree, can_create) +{ + ArrayStore store(10); + IndexedTree tree = IndexedTree(store, 10); + EXPECT_EQ(tree.size(), 1ULL); + + NullifierMemoryTree memdb(10); + EXPECT_EQ(memdb.root(), tree.root()); +} + +TEST(stdlib_indexed_tree, test_size) +{ + ArrayStore store(32); + auto db = IndexedTree(store, 32); + + // We assume that the first leaf is already filled with (0, 0, 0). + EXPECT_EQ(db.size(), 1ULL); + + // Add a new non-zero leaf at index 1. + db.add_value(30); + EXPECT_EQ(db.size(), 2ULL); + + // Add third. + db.add_value(10); + EXPECT_EQ(db.size(), 3ULL); + + // Add forth. + db.add_value(20); + EXPECT_EQ(db.size(), 4ULL); +} + +TEST(stdlib_indexed_tree, test_get_hash_path) +{ + NullifierMemoryTree memdb(10); + + ArrayStore store(10); + auto db = IndexedTree(store, 10); + + EXPECT_EQ(memdb.root(), db.root()); + + EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); + + memdb.update_element(VALUES[512]); + db.add_value(VALUES[512]); + + EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); + + for (size_t i = 0; i < 512; ++i) { + memdb.update_element(VALUES[i]); + db.add_value(VALUES[i]); + } + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); +} + +TEST(stdlib_indexed_tree, test_batch_insert) +{ + const size_t batch_size = 16; + const size_t num_batches = 16; + size_t depth = 10; + NullifierMemoryTree memdb(depth, batch_size); + + ArrayStore store1(depth); + IndexedTree tree1 = + IndexedTree(store1, depth, batch_size); + + ArrayStore store2(depth); + IndexedTree tree2 = + IndexedTree(store2, depth, batch_size); + + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t i = 0; i < num_batches; i++) { + std::vector batch; + std::vector memory_tree_hash_paths; + for (size_t j = 0; j < batch_size; j++) { + batch.push_back(fr(random_engine.get_random_uint256())); + fr_hash_path path = memdb.update_element(batch[j]); + memory_tree_hash_paths.push_back(path); + } + std::vector tree1_hash_paths = tree1.add_or_update_values(batch, true); + std::vector tree2_hash_paths = tree2.add_or_update_values(batch); + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t j = 0; j < batch_size; j++) { + EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); + // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); + } + } +} + +fr hash_leaf(const indexed_leaf& leaf) +{ + return HashPolicy::hash(leaf.get_hash_inputs()); +} + +bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const size_t idx) +{ + auto current = hash_leaf(leaf_value); + size_t depth_ = path.size(); + size_t index = idx; + for (size_t i = 0; i < depth_; ++i) { + fr left = (index & 1) ? path[i].first : current; + fr right = (index & 1) ? current : path[i].second; + current = HashPolicy::hash_pair(left, right); + index >>= 1; + } + return current == root; +} + +TEST(stdlib_indexed_tree, test_indexed_memory) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(tree.get_leaf(0), zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.add_value(30); + EXPECT_EQ(tree.size(), 2); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.add_value(10); + EXPECT_EQ(tree.size(), 3); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 1, 30 })); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.add_value(20); + EXPECT_EQ(tree.size(), 4); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + + // Adding the same value must not affect anything + // tree.update_element(20); + // EXPECT_EQ(tree.get_leaves().size(), 4); + // EXPECT_EQ(tree.get_leaves()[0], hash_leaf({ 0, 2, 10 })); + // EXPECT_EQ(tree.get_leaves()[1], hash_leaf({ 30, 0, 0 })); + // EXPECT_EQ(tree.get_leaves()[2], hash_leaf({ 10, 3, 20 })); + // EXPECT_EQ(tree.get_leaves()[3], hash_leaf({ 20, 1, 30 })); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.add_value(50); + EXPECT_EQ(tree.size(), 5); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(4)), hash_leaf({ 50, 0, 0 })); + + // Manually compute the node values + auto e000 = hash_leaf(tree.get_leaf(0)); + auto e001 = hash_leaf(tree.get_leaf(1)); + auto e010 = hash_leaf(tree.get_leaf(2)); + auto e011 = hash_leaf(tree.get_leaf(3)); + auto e100 = hash_leaf(tree.get_leaf(4)); + auto e101 = hash_leaf({ 0, 0, 0 }); + auto e110 = hash_leaf({ 0, 0, 0 }); + auto e111 = hash_leaf({ 0, 0, 0 }); + + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected); + EXPECT_EQ(tree.get_hash_path(1), expected); + fr_hash_path expected2 = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected2); + EXPECT_EQ(tree.get_hash_path(3), expected2); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} + +TEST(stdlib_indexed_tree, test_indexed_tree) +{ + // Create a depth-8 indexed merkle tree + constexpr size_t depth = 8; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); + + // Add 20 random values to the tree + for (size_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + tree.add_value(value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + std::vector differences; + for (size_t i = 0; i < size_t(tree.size()); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + auto hash_path = tree.get_hash_path(index); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp new file mode 100644 index 000000000000..cd29811455dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp @@ -0,0 +1,49 @@ +#include "leaves_cache.hpp" + +namespace bb::crypto::merkle_tree { + +index_t LeavesCache::get_size() const +{ + return index_t(leaves_.size()); +} + +std::pair LeavesCache::find_low_value(const fr& new_value) const +{ + std::map::const_iterator it = indices_.lower_bound(new_value); + if (it == indices_.end()) { + // there is no element >= the requested value. + // decrement the iterator to get the value preceeding the requested value + --it; + return std::make_pair(false, it->second); + } + if (it->first == uint256_t(new_value)) { + // the value is already present and the iterator points to it + return std::make_pair(true, it->second); + } + // the iterator points to the element immediately larger than the requested value + --it; + // it now points to the value less than that requested + return std::make_pair(false, it->second); +} +indexed_leaf LeavesCache::get_leaf(const index_t& index) const +{ + ASSERT(index >= 0 && index < leaves_.size()); + return leaves_[size_t(index)]; +} +void LeavesCache::set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index) +{ + if (index >= leaves_.size()) { + leaves_.resize(size_t(index + 1)); + } + leaves_[size_t(index)] = leaf; + if (add_to_index) { + indices_[uint256_t(leaf.value)] = index; + } +} +void LeavesCache::append_leaf(const indexed_leaf& leaf) +{ + index_t next_index = leaves_.size(); + set_at_index(next_index, leaf, true); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp new file mode 100644 index 000000000000..b38f556807ea --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp @@ -0,0 +1,27 @@ +#pragma once +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "indexed_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +/** + * @brief Used to facilitate testing of the IndexedTree. Stores leaves in memory with an index for O(logN) retrieval of + * 'low leaves' + * + */ +class LeavesCache { + public: + index_t get_size() const; + std::pair find_low_value(const bb::fr& new_value) const; + indexed_leaf get_leaf(const index_t& index) const; + void set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index); + void append_leaf(const indexed_leaf& leaf); + + private: + std::map indices_; + std::vector leaves_; +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp index ddb743b6e8ec..4d38e4311c8e 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp @@ -1,14 +1,16 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/hash/pedersen/pedersen.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include "hash_path.hpp" -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -template using bit_vector = std::vector>; +template using bit_vector = std::vector>; +template using field_t = bb::stdlib::field_t; +template using bool_t = bb::stdlib::bool_t; /** * Computes the new merkle root if the subtree is correctly inserted at a specified index in a Merkle tree. * @@ -42,9 +44,9 @@ field_t compute_subtree_root(hash_path const& hashes, field_t left = field_t::conditional_assign(path_bit, hashes[i].first, current); field_t right = field_t::conditional_assign(path_bit, current, hashes[i].second); if (is_updating_tree) { - current = pedersen_hash::hash({ left, right }, 0); + current = bb::stdlib::pedersen_hash::hash({ left, right }, 0); } else { - current = pedersen_hash::hash_skip_field_validation({ left, right }, 0); + current = bb::stdlib::pedersen_hash::hash_skip_field_validation({ left, right }, 0); } } @@ -143,7 +145,7 @@ void assert_check_membership(field_t const& root, bool const is_updating_tree = false, std::string const& msg = "assert_check_membership") { - auto exists = stdlib::merkle_tree::check_membership(root, hashes, value, index, is_updating_tree); + auto exists = check_membership(root, hashes, value, index, is_updating_tree); exists.assert_equal(true, msg); } @@ -255,7 +257,7 @@ template field_t compute_tree_root(std::vector 1) { std::vector> next_layer(layer.size() / 2); for (size_t i = 0; i < next_layer.size(); ++i) { - next_layer[i] = pedersen_hash::hash({ layer[i * 2], layer[i * 2 + 1] }); + next_layer[i] = bb::stdlib::pedersen_hash::hash({ layer[i * 2], layer[i * 2 + 1] }); } layer = std::move(next_layer); } @@ -316,7 +318,7 @@ void batch_update_membership(field_t const& new_root, new_root, rollup_root, old_root, old_path, zero_subtree_root, start_index.decompose_into_bits(), height, msg); } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree namespace bb::stdlib { /** @@ -331,8 +333,8 @@ template static void generate_merkle_membership_test_circuit( using namespace stdlib; using field_ct = field_t; using witness_ct = witness_t; - using MemStore = merkle_tree::MemoryStore; - using MerkleTree_ct = merkle_tree::MerkleTree; + using MemStore = crypto::merkle_tree::MemoryStore; + using MerkleTree_ct = crypto::merkle_tree::MerkleTree; MemStore store; const size_t tree_depth = 7; @@ -348,8 +350,11 @@ template static void generate_merkle_membership_test_circuit( auto idx_ct = field_ct(witness_ct(&builder, fr(idx))).decompose_into_bits(); auto value_ct = field_ct(value); - merkle_tree::check_membership( - root_ct, merkle_tree::create_witness_hash_path(builder, merkle_tree.get_hash_path(idx)), value_ct, idx_ct); + crypto::merkle_tree::check_membership( + root_ct, + crypto::merkle_tree::create_witness_hash_path(builder, merkle_tree.get_hash_path(idx)), + value_ct, + idx_ct); } } } // namespace bb::stdlib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp index 1d8d06820e74..3975c4abccd7 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp @@ -8,7 +8,7 @@ #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp" using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::stdlib; namespace { @@ -17,14 +17,15 @@ auto& engine = numeric::get_debug_randomness(); using Builder = UltraCircuitBuilder; -using bool_ct = bool_t; -using field_ct = field_t; -using witness_ct = witness_t; +using bool_ct = bb::stdlib::bool_t; +using field_ct = bb::stdlib::field_t; +using witness_ct = bb::stdlib::witness_t; +using tree = MerkleTree; -TEST(stdlib_merkle_tree, test_check_membership) +TEST(crypto_merkle_tree, test_check_membership) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); // Check membership at index 0. @@ -48,10 +49,10 @@ TEST(stdlib_merkle_tree, test_check_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_batch_update_membership) +TEST(crypto_merkle_tree, test_batch_update_membership) { MemoryStore store; - MerkleTree db(store, 4); + tree db(store, 4); auto builder = Builder(); // Fill in an arbitrary value at i = 2. db.update_element(2, fr::random_element()); @@ -74,10 +75,10 @@ TEST(stdlib_merkle_tree, test_batch_update_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_assert_check_membership) +TEST(crypto_merkle_tree, test_assert_check_membership) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); auto zero = field_ct(witness_ct(&builder, fr::zero())).decompose_into_bits(); @@ -91,10 +92,10 @@ TEST(stdlib_merkle_tree, test_assert_check_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_assert_check_membership_fail) +TEST(crypto_merkle_tree, test_assert_check_membership_fail) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -109,11 +110,11 @@ TEST(stdlib_merkle_tree, test_assert_check_membership_fail) EXPECT_EQ(result, false); } // To test whether both old hash path and new hash path works for the same Merkle tree -TEST(stdlib_merkle_tree, test_update_members) +TEST(crypto_merkle_tree, test_update_members) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -137,7 +138,7 @@ TEST(stdlib_merkle_tree, test_update_members) } { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -161,13 +162,13 @@ TEST(stdlib_merkle_tree, test_update_members) } } -TEST(stdlib_merkle_tree, test_tree) +TEST(crypto_merkle_tree, test_tree) { size_t depth = 3; size_t num = 1UL << depth; MemoryStore store; - MerkleTree db(store, depth); - MemoryTree mem_tree(depth); + tree db(store, depth); + MemoryTree mem_tree(depth); auto builder = Builder(); @@ -183,11 +184,11 @@ TEST(stdlib_merkle_tree, test_tree) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_update_memberships) +TEST(crypto_merkle_tree, test_update_memberships) { constexpr size_t depth = 4; MemoryStore store; - MerkleTree tree(store, depth); + tree tree(store, depth); auto builder = Builder(); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp index ee1f4196a740..a1b5dfec6691 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp @@ -4,7 +4,7 @@ #include #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { class MemoryStore { public: @@ -84,4 +84,4 @@ class MemoryStore { std::set deletes_; }; -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp new file mode 100644 index 000000000000..e85e613aa9dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp @@ -0,0 +1,114 @@ +#pragma once +#include "hash_path.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * A MemoryTree is structured as follows: + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + */ +template class MemoryTree { + public: + MemoryTree(size_t depth); + + fr_hash_path get_hash_path(size_t index); + + fr_sibling_path get_sibling_path(size_t index); + + fr update_element(size_t index, fr const& value); + + fr root() const { return root_; } + + public: + size_t depth_; + size_t total_size_; + bb::fr root_; + std::vector hashes_; +}; + +template +MemoryTree::MemoryTree(size_t depth) + : depth_(depth) +{ + + ASSERT(depth_ >= 1 && depth <= 20); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree. + auto current = fr(0); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = HashingPolicy::hash_pair(current, current); + } + + root_ = current; +} + +template fr_hash_path MemoryTree::get_hash_path(size_t index) +{ + fr_hash_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; ++i) { + index -= index & 0x1; + path[i] = std::make_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr_sibling_path MemoryTree::get_sibling_path(size_t index) +{ + fr_sibling_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; i++) { + if (index % 2 == 0) { + path[i] = hashes_[offset + index + 1]; + } else { + path[i] = hashes_[offset + index - 1]; + } + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr MemoryTree::update_element(size_t index, fr const& value) +{ + size_t offset = 0; + size_t layer_size = total_size_; + fr current = value; + for (size_t i = 0; i < depth_; ++i) { + hashes_[offset + index] = current; + index &= (~0ULL) - 1; + current = HashingPolicy::hash_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + root_ = current; + return root_; +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp similarity index 74% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp index cfbf5e8a6a90..56cca4b03bc9 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp @@ -2,7 +2,9 @@ #include using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = PedersenHashPolicy; static std::vector VALUES = []() { std::vector values(4); @@ -12,17 +14,17 @@ static std::vector VALUES = []() { return values; }(); -TEST(stdlib_merkle_tree, test_memory_store) +TEST(crypto_merkle_tree, test_memory_store) { fr e00 = 0; fr e01 = VALUES[1]; fr e02 = VALUES[2]; fr e03 = VALUES[3]; - fr e10 = hash_pair_native(e00, e01); - fr e11 = hash_pair_native(e02, e03); - fr root = hash_pair_native(e10, e11); + fr e10 = HashPolicy::hash_pair(e00, e01); + fr e11 = HashPolicy::hash_pair(e02, e03); + fr root = HashPolicy::hash_pair(e10, e11); - MemoryTree db(2); + MemoryTree db(2); for (size_t i = 0; i < 4; ++i) { db.update_element(i, VALUES[i]); } @@ -43,17 +45,17 @@ TEST(stdlib_merkle_tree, test_memory_store) EXPECT_EQ(db.root(), root); } -TEST(stdlib_merkle_tree, test_memory_store_sibling_path) +TEST(crypto_merkle_tree, test_memory_store_sibling_path) { fr e00 = 0; fr e01 = VALUES[1]; fr e02 = VALUES[2]; fr e03 = VALUES[3]; - fr e10 = hash_pair_native(e00, e01); - fr e11 = hash_pair_native(e02, e03); - fr root = hash_pair_native(e10, e11); + fr e10 = HashPolicy::hash_pair(e00, e01); + fr e11 = HashPolicy::hash_pair(e02, e03); + fr root = HashPolicy::hash_pair(e10, e11); - MemoryTree db(2); + MemoryTree db(2); for (size_t i = 0; i < 4; ++i) { db.update_element(i, VALUES[i]); } diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp similarity index 67% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp index 6ea45c94e756..576f5bfcced4 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp @@ -1,16 +1,104 @@ -#include "merkle_tree.hpp" +#pragma once #include "barretenberg/common/net.hpp" #include "barretenberg/numeric/bitop/count_leading_zeros.hpp" #include "barretenberg/numeric/bitop/keep_n_lsb.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" -#include "hash.hpp" -#include "memory_store.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "hash_path.hpp" +#include "merkle_tree.hpp" #include #include -namespace bb::stdlib::merkle_tree { - -using namespace bb; +namespace bb::crypto::merkle_tree { + +class MemoryStore; + +template class MerkleTree { + public: + typedef uint256_t index_t; + + MerkleTree(Store& store, size_t depth, uint8_t tree_id = 0); + MerkleTree(MerkleTree const& other) = delete; + MerkleTree(MerkleTree&& other); + ~MerkleTree(); + + fr_hash_path get_hash_path(index_t index); + + fr_sibling_path get_sibling_path(index_t index); + + fr update_element(index_t index, fr const& value); + + fr root() const; + + size_t depth() const { return depth_; } + + index_t size() const; + + protected: + void load_metadata(); + + /** + * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. + * + * @param height: The tree depth + * @param index: the index of the non-empty leaf + * @param value: the value to be stored in the non-empty leaf + * + * @see Check full documentation: https://hackmd.io/2zyJc6QhRuugyH8D78Tbqg?view + */ + fr update_element(fr const& root, fr const& value, index_t index, size_t height); + + fr get_element(fr const& root, index_t index, size_t height); + + /** + * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. + * + * @param height: The tree depth + * @param index: the index of the non-empty leaf + * @param value: the value to be stored in the non-empty leaf + */ + fr compute_zero_path_hash(size_t height, index_t index, fr const& value); + + /** + * Given child nodes `a` and `b` and index of `a`, compute their parent node `p` and store [p : (a, b)]. + * + * @param a_index: the index of the child node `a` + * @param a: child node + * @param b: child node + * @param height: the height of the parent node + */ + fr binary_put(index_t a_index, fr const& a, fr const& b, size_t height); + + fr fork_stump( + fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t stump_height); + + /** + * Stores a parent node and child nodes in the database as [key : (left, right)]. + * + * @param key: The node value to be stored as key + * @param left: the left child node + * @param right: the right child node + */ + void put(fr const& key, fr const& left, fr const& right); + + /** + * Stores a stump [key : (value, index, true)] in the memory. + * The additional byte `true` is to denote this is a stump. + * + * @param key: The node value to be stored as key + * @param value: value of the non-empty leaf in the stump + * @param index: the index of the non-empty leaf in the stump + */ + void put_stump(fr const& key, index_t index, fr const& value); + + void remove(fr const& key); + + protected: + Store& store_; + std::vector zero_hashes_; + size_t depth_; + uint8_t tree_id_; +}; // Size of merkle tree nodes in bytes. constexpr size_t REGULAR_NODE_SIZE = 64; @@ -21,8 +109,8 @@ template inline bool bit_set(T const& index, size_t i) return bool((index >> i) & 0x1); } -template -MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) +template +MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) : store_(store) , depth_(depth) , tree_id_(tree_id) @@ -34,29 +122,30 @@ MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) auto current = fr(0); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; - current = hash_pair_native(current, current); + current = HashingPolicy::hash_pair(current, current); } } -template -MerkleTree::MerkleTree(MerkleTree&& other) +template +MerkleTree::MerkleTree(MerkleTree&& other) : store_(other.store_) , zero_hashes_(std::move(other.zero_hashes_)) , depth_(other.depth_) , tree_id_(other.tree_id_) {} -template MerkleTree::~MerkleTree() {} +template MerkleTree::~MerkleTree() {} -template fr MerkleTree::root() const +template fr MerkleTree::root() const { std::vector root; std::vector key = { tree_id_ }; bool status = store_.get(key, root); - return status ? from_buffer(root) : hash_pair_native(zero_hashes_.back(), zero_hashes_.back()); + return status ? from_buffer(root) : HashingPolicy::hash_pair(zero_hashes_.back(), zero_hashes_.back()); } -template typename MerkleTree::index_t MerkleTree::size() const +template +typename MerkleTree::index_t MerkleTree::size() const { std::vector size_buf; std::vector key = { tree_id_ }; @@ -64,7 +153,8 @@ template typename MerkleTree::index_t MerkleTree: return status ? from_buffer(size_buf, 32) : 0; } -template fr_hash_path MerkleTree::get_hash_path(index_t index) +template +fr_hash_path MerkleTree::get_hash_path(index_t index) { fr_hash_path path(depth_); @@ -105,7 +195,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = hash_pair_native(path[j].first, path[j].second); + current = HashingPolicy::hash_pair(path[j].first, path[j].second); } } else { // Requesting path to a different, independent element. @@ -125,7 +215,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = hash_pair_native(path[j].first, path[j].second); + current = HashingPolicy::hash_pair(path[j].first, path[j].second); } } break; @@ -135,7 +225,8 @@ template fr_hash_path MerkleTree::get_hash_path(index_t return path; } -template fr_sibling_path MerkleTree::get_sibling_path(index_t index) +template +fr_sibling_path MerkleTree::get_sibling_path(index_t index) { fr_sibling_path path(depth_); @@ -194,7 +285,8 @@ template fr_sibling_path MerkleTree::get_sibling_path(in return path; } -template fr MerkleTree::update_element(index_t index, fr const& value) +template +fr MerkleTree::update_element(index_t index, fr const& value) { auto leaf = value; using serialize::write; @@ -214,18 +306,19 @@ template fr MerkleTree::update_element(index_t index, fr return r; } -template fr MerkleTree::binary_put(index_t a_index, fr const& a, fr const& b, size_t height) +template +fr MerkleTree::binary_put(index_t a_index, fr const& a, fr const& b, size_t height) { bool a_is_right = bit_set(a_index, height - 1); auto left = a_is_right ? b : a; auto right = a_is_right ? a : b; - auto key = hash_pair_native(left, right); + auto key = HashingPolicy::hash_pair(left, right); put(key, left, right); return key; } -template -fr MerkleTree::fork_stump( +template +fr MerkleTree::fork_stump( fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t common_height) { if (height == common_height) { @@ -250,8 +343,8 @@ fr MerkleTree::fork_stump( } } -template -fr MerkleTree::update_element(fr const& root, fr const& value, index_t index, size_t height) +template +fr MerkleTree::update_element(fr const& root, fr const& value, index_t index, size_t height) { // Base layer of recursion at height = 0. if (height == 0) { @@ -297,7 +390,7 @@ fr MerkleTree::update_element(fr const& root, fr const& value, index_t in } else { left = subtree_root; } - auto new_root = hash_pair_native(left, right); + auto new_root = HashingPolicy::hash_pair(left, right); put(new_root, left, right); // Remove the old node only while rolling back in recursion. @@ -308,7 +401,8 @@ fr MerkleTree::update_element(fr const& root, fr const& value, index_t in } } -template fr MerkleTree::compute_zero_path_hash(size_t height, index_t index, fr const& value) +template +fr MerkleTree::compute_zero_path_hash(size_t height, index_t index, fr const& value) { fr current = value; for (size_t i = 0; i < height; ++i) { @@ -321,12 +415,13 @@ template fr MerkleTree::compute_zero_path_hash(size_t he right = zero_hashes_[i]; left = current; } - current = hash_pair_native(left, right); + current = HashingPolicy::hash_pair(left, right); } return current; } -template void MerkleTree::put(fr const& key, fr const& left, fr const& right) +template +void MerkleTree::put(fr const& key, fr const& left, fr const& right) { std::vector value; write(value, left); @@ -334,7 +429,8 @@ template void MerkleTree::put(fr const& key, fr const& l store_.put(key.to_buffer(), value); } -template void MerkleTree::put_stump(fr const& key, index_t index, fr const& value) +template +void MerkleTree::put_stump(fr const& key, index_t index, fr const& value) { std::vector buf; write(buf, value); @@ -344,11 +440,9 @@ template void MerkleTree::put_stump(fr const& key, index store_.put(key.to_buffer(), buf); } -template void MerkleTree::remove(fr const& key) +template void MerkleTree::remove(fr const& key) { store_.del(key.to_buffer()); } -template class MerkleTree; - -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp similarity index 78% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp index 55bd530c2161..085f97555abe 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp @@ -6,7 +6,7 @@ #include "memory_tree.hpp" using namespace bb::stdlib; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using Builder = UltraCircuitBuilder; @@ -25,13 +25,13 @@ static std::vector VALUES = []() { return values; }(); -TEST(stdlib_merkle_tree, test_kv_memory_vs_memory_consistency) +TEST(crypto_merkle_tree, test_kv_memory_vs_memory_consistency) { constexpr size_t depth = 10; - MemoryTree memdb(depth); + MemoryTree memdb(depth); MemoryStore store; - MerkleTree db(store, depth); + MerkleTree db(store, depth); std::vector indicies(1 << depth); std::iota(indicies.begin(), indicies.end(), 0); @@ -53,10 +53,10 @@ TEST(stdlib_merkle_tree, test_kv_memory_vs_memory_consistency) EXPECT_EQ(db.root(), memdb.root()); } -TEST(stdlib_merkle_tree, test_size) +TEST(crypto_merkle_tree, test_size) { MemoryStore store; - auto db = MerkleTree(store, 256); + auto db = MerkleTree(store, 256); EXPECT_EQ(db.size(), 0ULL); @@ -81,12 +81,12 @@ TEST(stdlib_merkle_tree, test_size) EXPECT_EQ(db.size(), 3ULL); } -TEST(stdlib_merkle_tree, test_get_hash_path) +TEST(crypto_merkle_tree, test_get_hash_path) { - MemoryTree memdb(10); + MemoryTree memdb(10); MemoryStore store; - auto db = MerkleTree(store, 10); + auto db = MerkleTree(store, 10); EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); @@ -103,12 +103,12 @@ TEST(stdlib_merkle_tree, test_get_hash_path) EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); } -TEST(stdlib_merkle_tree, test_get_sibling_path) +TEST(crypto_merkle_tree, test_get_sibling_path) { - MemoryTree memdb(10); + MemoryTree memdb(10); MemoryStore store; - auto db = MerkleTree(store, 10); + auto db = MerkleTree(store, 10); EXPECT_EQ(memdb.get_sibling_path(512), db.get_sibling_path(512)); @@ -125,11 +125,11 @@ TEST(stdlib_merkle_tree, test_get_sibling_path) EXPECT_EQ(db.get_sibling_path(512), memdb.get_sibling_path(512)); } -TEST(stdlib_merkle_tree, test_get_hash_path_layers) +TEST(crypto_merkle_tree, test_get_hash_path_layers) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_hash_path(1); db.update_element(0, VALUES[1]); @@ -142,7 +142,7 @@ TEST(stdlib_merkle_tree, test_get_hash_path_layers) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_hash_path(7); db.update_element(0x0, VALUES[1]); @@ -154,11 +154,11 @@ TEST(stdlib_merkle_tree, test_get_hash_path_layers) } } -TEST(stdlib_merkle_tree, test_get_sibling_path_layers) +TEST(crypto_merkle_tree, test_get_sibling_path_layers) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_sibling_path(1); db.update_element(0, VALUES[1]); @@ -171,7 +171,7 @@ TEST(stdlib_merkle_tree, test_get_sibling_path_layers) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_sibling_path(7); db.update_element(0x0, VALUES[1]); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp similarity index 76% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp index d1bd3d310d87..1ac402ac3102 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -1,10 +1,10 @@ #pragma once #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/stdlib/hash/poseidon2/poseidon2.hpp" -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -using namespace bb; typedef uint256_t index_t; struct nullifier_leaf { @@ -22,14 +22,16 @@ struct nullifier_leaf { return os; } - bb::fr hash() const { return stdlib::merkle_tree::hash_native({ value, nextIndex, nextValue }); } + std::vector get_hash_inputs() const { return std::vector{ value, nextIndex, nextValue }; } + + static nullifier_leaf zero() { return nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; } }; /** * @brief Wrapper for the Nullifier leaf class that allows for 0 values * */ -class WrappedNullifierLeaf { +template class WrappedNullifierLeaf { public: // Initialize with a nullifier leaf @@ -70,21 +72,26 @@ class WrappedNullifierLeaf { * * @return bb::fr */ - bb::fr hash() const { return data.has_value() ? data.value().hash() : bb::fr::zero(); } + bb::fr hash() const + { + return data.has_value() ? HashingPolicy::hash(data.value().get_hash_inputs()) : bb::fr::zero(); + } /** * @brief Generate a zero leaf (call the constructor with no arguments) * * @return NullifierLeaf */ - static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } + static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } private: // Underlying data std::optional data; }; -inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) +template +inline std::pair find_closest_leaf(std::vector> const& leaves_, + fr const& new_value) { std::vector diff; bool repeated = false; @@ -111,4 +118,4 @@ inline std::pair find_closest_leaf(std::vector(it - diff.begin()), repeated); } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp new file mode 100644 index 000000000000..5b9fdb94ee2d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -0,0 +1,174 @@ +#pragma once +#include "../hash.hpp" +#include "../memory_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * An NullifierMemoryTree is structured just like a usual merkle tree: + * + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + * + * 1. Initial state: + * + * # + * + * # # + * + * # # # # + * + * # # # # # # # # + * + * index 0 1 2 3 4 5 6 7 + * + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + * + * 2. Add new leaf with value 30 + * + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + * + * 3. Add new leaf with value 10 + * + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + * + * 4. Add new leaf with value 20 + * + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + * + * 5. Add new leaf with value 50 + * + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ +template class NullifierMemoryTree : public MemoryTree { + + public: + NullifierMemoryTree(size_t depth, size_t initial_size = 1); + + using MemoryTree::get_hash_path; + using MemoryTree::root; + using MemoryTree::update_element; + + fr_hash_path update_element(fr const& value); + + const std::vector& get_hashes() { return hashes_; } + const WrappedNullifierLeaf get_leaf(size_t index) + { + return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf(nullifier_leaf::zero()); + } + const std::vector>& get_leaves() { return leaves_; } + + protected: + using MemoryTree::depth_; + using MemoryTree::hashes_; + using MemoryTree::root_; + using MemoryTree::total_size_; + std::vector> leaves_; +}; + +template +NullifierMemoryTree::NullifierMemoryTree(size_t depth, size_t initial_size) + : MemoryTree(depth) +{ + ASSERT(depth_ >= 1 && depth <= 32); + ASSERT(initial_size > 0); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree and fill with 0 hashes. + auto current = WrappedNullifierLeaf(nullifier_leaf::zero()).hash(); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = HashingPolicy::hash_pair(current, current); + } + + // Insert the initial leaves + for (size_t i = 0; i < initial_size; i++) { + auto initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + leaves_.push_back(initial_leaf); + } + + leaves_[initial_size - 1] = WrappedNullifierLeaf( + nullifier_leaf{ .value = leaves_[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + + for (size_t i = 0; i < initial_size; ++i) { + update_element(i, leaves_[i].hash()); + } +} + +template fr_hash_path NullifierMemoryTree::update_element(fr const& value) +{ + // Find the leaf with the value closest and less than `value` + + // If value is 0 we simply append 0 a null NullifierLeaf to the tree + fr_hash_path hash_path; + if (value == 0) { + auto zero_leaf = WrappedNullifierLeaf::zero(); + hash_path = get_hash_path(leaves_.size() - 1); + leaves_.push_back(zero_leaf); + update_element(leaves_.size() - 1, zero_leaf.hash()); + return hash_path; + } + + size_t current; + bool is_already_present; + std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); + + nullifier_leaf current_leaf = leaves_[current].unwrap(); + nullifier_leaf new_leaf = { .value = value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + current_leaf.nextIndex = leaves_.size(); + current_leaf.nextValue = value; + + leaves_[current].set(current_leaf); + + // Insert the new leaf with (nextIndex, nextValue) of the current leaf + leaves_.push_back(new_leaf); + } + + hash_path = get_hash_path(current); + // Update the old leaf in the tree + auto old_leaf_hash = HashingPolicy::hash(current_leaf.get_hash_inputs()); + size_t old_leaf_index = current; + auto root = update_element(old_leaf_index, old_leaf_hash); + + // Insert the new leaf in the tree + auto new_leaf_hash = HashingPolicy::hash(new_leaf.get_hash_inputs()); + size_t new_leaf_index = is_already_present ? old_leaf_index : leaves_.size() - 1; + root = update_element(new_leaf_index, new_leaf_hash); + + return hash_path; +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp similarity index 67% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp index eecce8b87af7..e33051822047 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -2,7 +2,10 @@ #include using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = PedersenHashPolicy; +using WrappedLeaf = WrappedNullifierLeaf; void print_tree(const size_t depth, std::vector hashes, std::string const& msg) { @@ -20,7 +23,7 @@ void print_tree(const size_t depth, std::vector hashes, std::string const& m bool check_hash_path(const fr& root, const fr_hash_path& path, const nullifier_leaf& leaf_value, const size_t idx) { - auto current = leaf_value.hash(); + auto current = WrappedLeaf(leaf_value).hash(); size_t depth_ = path.size(); size_t index = idx; for (size_t i = 0; i < depth_; ++i) { @@ -36,7 +39,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) { // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); /** * Intial state: @@ -49,7 +52,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + EXPECT_EQ(tree.get_leaves()[0].unwrap(), zero_leaf); /** * Add new value 30: @@ -62,8 +65,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -76,9 +79,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -91,18 +94,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); /** * Add new value 50: @@ -115,11 +118,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 4, 50 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 4, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 50, 0, 0 }).hash()); // Manually compute the node values auto e000 = tree.get_leaves()[0].hash(); @@ -127,18 +130,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) auto e010 = tree.get_leaves()[2].hash(); auto e011 = tree.get_leaves()[3].hash(); auto e100 = tree.get_leaves()[4].hash(); - auto e101 = WrappedNullifierLeaf::zero().hash(); - auto e110 = WrappedNullifierLeaf::zero().hash(); - auto e111 = WrappedNullifierLeaf::zero().hash(); + auto e101 = WrappedLeaf(zero_leaf).hash(); + auto e110 = WrappedLeaf(zero_leaf).hash(); + auto e111 = WrappedLeaf(zero_leaf).hash(); - auto e00 = hash_pair_native(e000, e001); - auto e01 = hash_pair_native(e010, e011); - auto e10 = hash_pair_native(e100, e101); - auto e11 = hash_pair_native(e110, e111); + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); - auto e0 = hash_pair_native(e00, e01); - auto e1 = hash_pair_native(e10, e11); - auto root = hash_pair_native(e0, e1); + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); // Check the hash path at index 2 and 3 // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) @@ -165,7 +168,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) { // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); /** * Intial state: @@ -176,7 +179,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - WrappedNullifierLeaf zero_leaf = WrappedNullifierLeaf({ 0, 0, 0 }); + WrappedLeaf zero_leaf = WrappedLeaf({ 0, 0, 0 }); EXPECT_EQ(tree.get_leaves().size(), 1); EXPECT_EQ(tree.get_leaves()[0], zero_leaf); @@ -191,8 +194,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -205,9 +208,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -220,18 +223,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); /** * Add new value 0: @@ -244,11 +247,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(0); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); /* * Add new value 0: @@ -261,12 +264,12 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(0); EXPECT_EQ(tree.get_leaves().size(), 6); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedLeaf::zero().hash()); /** * Add new value 50: @@ -279,14 +282,14 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 7); - EXPECT_EQ(tree.get_leaf(0).hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaf(1).hash(), WrappedNullifierLeaf({ 30, 6, 50 }).hash()); - EXPECT_EQ(tree.get_leaf(2).hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaf(3).hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaf(4).hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaf(5).hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaf(6).hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaf(7).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(0).hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaf(1).hash(), WrappedLeaf({ 30, 6, 50 }).hash()); + EXPECT_EQ(tree.get_leaf(2).hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaf(3).hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaf(4).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(5).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(6).hash(), WrappedLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaf(7).hash(), zero_leaf.hash()); // Manually compute the node values auto e000 = tree.get_leaf(0).hash(); @@ -298,14 +301,22 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) auto e110 = tree.get_leaf(6).hash(); auto e111 = tree.get_leaf(7).hash(); - auto e00 = hash_pair_native(e000, e001); - auto e01 = hash_pair_native(e010, e011); - auto e10 = hash_pair_native(e100, e101); - auto e11 = hash_pair_native(e110, e111); + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); - auto e0 = hash_pair_native(e00, e01); - auto e1 = hash_pair_native(e10, e11); - auto root = hash_pair_native(e0, e1); + fr_hash_path expected1 = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected1); + EXPECT_EQ(tree.get_hash_path(1), expected1); // Check the hash path at index 2 and 3 // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) @@ -331,11 +342,11 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) { // Create a depth-8 indexed merkle tree constexpr size_t depth = 8; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0].hash(), zero_leaf.hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf(zero_leaf).hash()); // Add 20 random values to the tree for (size_t i = 0; i < 20; i++) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp similarity index 51% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp index 502ea05c1b13..95ee0ea72abb 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -9,43 +9,48 @@ #include #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -template inline bool bit_set(T const& index, size_t i) -{ - return bool((index >> i) & 0x1); -} - -template -NullifierTree::NullifierTree(Store& store, size_t depth, uint8_t tree_id) - : MerkleTree(store, depth, tree_id) +template +NullifierTree::NullifierTree(Store& store, size_t depth, size_t initial_size, uint8_t tree_id) + : MerkleTree(store, depth, tree_id) { ASSERT(depth_ >= 1 && depth <= 256); + ASSERT(initial_size > 0); zero_hashes_.resize(depth); - // Compute the zero values at each layer. - // Insert the zero leaf to the `leaves` and also to the tree at index 0. - WrappedNullifierLeaf initial_leaf = - WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); - leaves.push_back(initial_leaf); - update_element(0, initial_leaf.hash()); - // Create the zero hashes for the tree - auto current = WrappedNullifierLeaf::zero().hash(); + auto current = + WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }).hash(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; - current = hash_pair_native(current, current); + current = HashingPolicy::hash_pair(current, current); + } + + // Insert the initial leaves + for (size_t i = 0; i < initial_size; i++) { + auto initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + leaves.push_back(initial_leaf); + } + + leaves[initial_size - 1] = WrappedNullifierLeaf( + nullifier_leaf{ .value = leaves[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + + for (size_t i = 0; i < initial_size; ++i) { + update_element(i, leaves[i].hash()); } } -template -NullifierTree::NullifierTree(NullifierTree&& other) - : MerkleTree(std::move(other)) +template +NullifierTree::NullifierTree(NullifierTree&& other) + : MerkleTree(std::move(other)) {} -template NullifierTree::~NullifierTree() {} +template NullifierTree::~NullifierTree() {} -template fr NullifierTree::update_element(fr const& value) +template +fr NullifierTree::update_element(fr const& value) { // Find the leaf with the value closest and less than `value` size_t current; @@ -53,7 +58,7 @@ template fr NullifierTree::update_element(fr const& valu std::tie(current, is_already_present) = find_closest_leaf(leaves, value); nullifier_leaf current_leaf = leaves[current].unwrap(); - WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( + WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( { .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }); if (!is_already_present) { // Update the current leaf to point it to the new leaf @@ -79,6 +84,6 @@ template fr NullifierTree::update_element(fr const& valu return r; } -template class NullifierTree; +template class NullifierTree; -} // namespace bb::stdlib::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp new file mode 100644 index 000000000000..ea727de6855f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp @@ -0,0 +1,37 @@ +#pragma once +#include "../hash.hpp" +#include "../merkle_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +template class NullifierTree : public MerkleTree { + public: + typedef uint256_t index_t; + + NullifierTree(Store& store, size_t depth, size_t initial_size = 1, uint8_t tree_id = 0); + NullifierTree(NullifierTree const& other) = delete; + NullifierTree(NullifierTree&& other); + ~NullifierTree(); + + using MerkleTree::get_hash_path; + using MerkleTree::root; + using MerkleTree::size; + using MerkleTree::depth; + + fr update_element(fr const& value); + + private: + using MerkleTree::update_element; + using MerkleTree::get_element; + using MerkleTree::compute_zero_path_hash; + + private: + using MerkleTree::store_; + using MerkleTree::zero_hashes_; + using MerkleTree::depth_; + using MerkleTree::tree_id_; + std::vector> leaves; +}; + +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp similarity index 81% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp index 99a1eeeffdeb..41ddc1670b29 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp @@ -6,17 +6,18 @@ #include "nullifier_memory_tree.hpp" using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; namespace { auto& engine = numeric::get_debug_randomness(); auto& random_engine = numeric::get_randomness(); } // namespace +const size_t NUM_VALUES = 1024; static std::vector VALUES = []() { - std::vector values(1024); - for (size_t i = 0; i < 1024; ++i) { - values[i] = fr(random_engine.get_random_uint64()); + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); } return values; }(); @@ -38,10 +39,12 @@ inline void print_tree(const size_t depth, std::vector hashes, std::string c TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) { constexpr size_t depth = 2; - NullifierMemoryTree memdb(depth); + NullifierMemoryTree memdb(depth); MemoryStore store; - NullifierTree db(store, depth); + NullifierTree db(store, depth); + + EXPECT_EQ(db.root(), memdb.root()); std::vector indicies(1 << depth); std::iota(indicies.begin(), indicies.end(), 0); @@ -66,7 +69,7 @@ TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) TEST(stdlib_nullifier_tree, test_size) { MemoryStore store; - auto db = NullifierTree(store, 256); + auto db = NullifierTree(store, 256); // We assume that the first leaf is already filled with (0, 0, 0). EXPECT_EQ(db.size(), 1ULL); @@ -90,10 +93,10 @@ TEST(stdlib_nullifier_tree, test_size) TEST(stdlib_nullifier_tree, test_get_hash_path) { - NullifierMemoryTree memdb(10); + NullifierMemoryTree memdb(10); MemoryStore store; - auto db = NullifierTree(store, 10); + auto db = NullifierTree(store, 10); EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); @@ -114,7 +117,7 @@ TEST(stdlib_nullifier_tree, test_get_hash_path_layers) { { MemoryStore store; - auto db = NullifierTree(store, 3); + auto db = NullifierTree(store, 3); auto before = db.get_hash_path(1); db.update_element(VALUES[1]); @@ -127,7 +130,7 @@ TEST(stdlib_nullifier_tree, test_get_hash_path_layers) { MemoryStore store; - auto db = NullifierTree(store, 3); + auto db = NullifierTree(store, 3); auto before = db.get_hash_path(7); db.update_element(VALUES[1]); diff --git a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp index 86f505105baf..b455ab5d6d50 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp @@ -5,23 +5,40 @@ extern "C" { -WASM_EXPORT void pedersen_hash(uint8_t const* inputs_buffer, uint32_t const* hash_index, uint8_t* output) +WASM_EXPORT void pedersen_hash(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output) { std::vector to_hash; read(inputs_buffer, to_hash); - crypto::GeneratorContext ctx; + bb::crypto::GeneratorContext ctx; ctx.offset = static_cast(ntohl(*hash_index)); - auto r = crypto::pedersen_hash::hash(to_hash, ctx); + auto r = bb::crypto::pedersen_hash::hash(to_hash, ctx); bb::fr::serialize_to_buffer(r, output); } -WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, uint8_t* output) +WASM_EXPORT void pedersen_hashes(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + bb::crypto::GeneratorContext ctx; + ctx.offset = static_cast(ntohl(*hash_index)); + const size_t numHashes = to_hash.size() / 2; + std::vector results; + size_t count = 0; + while (count < numHashes) { + auto r = bb::crypto::pedersen_hash::hash({ to_hash[count * 2], to_hash[count * 2 + 1] }, ctx); + results.push_back(r); + ++count; + } + write(output, results); +} + +WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, bb::fr::out_buf output) { std::vector to_hash; read(input_buffer, to_hash); - crypto::GeneratorContext ctx; + bb::crypto::GeneratorContext ctx; ctx.offset = static_cast(ntohl(*hash_index)); - auto r = crypto::pedersen_hash::hash_buffer(to_hash, ctx); + auto r = bb::crypto::pedersen_hash::hash_buffer(to_hash, ctx); bb::fr::serialize_to_buffer(r, output); } } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp index b3d8d50c30dc..ecd51260201e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp @@ -5,9 +5,7 @@ extern "C" { -using namespace bb; - -WASM_EXPORT void pedersen_hash(fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, fr::out_buf output); - -WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, fr::out_buf output); +WASM_EXPORT void pedersen_hash(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output); +WASM_EXPORT void pedersen_hashes(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output); +WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, bb::fr::out_buf output); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp new file mode 100644 index 000000000000..bd43c64915b2 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp @@ -0,0 +1,32 @@ +#include "c_bind.hpp" +#include "barretenberg/common/mem.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" +#include "poseidon2.hpp" + +extern "C" { + +WASM_EXPORT void poseidon_hash(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + auto r = bb::crypto::Poseidon2::hash(to_hash); + bb::fr::serialize_to_buffer(r, output); +} + +WASM_EXPORT void poseidon_hashes(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + const size_t numHashes = to_hash.size() / 2; + std::vector results; + size_t count = 0; + while (count < numHashes) { + auto r = bb::crypto::Poseidon2::hash( + { to_hash[count * 2], to_hash[count * 2 + 1] }); + results.push_back(r); + ++count; + } + write(output, results); +} +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp new file mode 100644 index 000000000000..c7dc58555485 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "barretenberg/common/wasm_export.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" + +extern "C" { + +WASM_EXPORT void poseidon_hash(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output); +WASM_EXPORT void poseidon_hashes(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt index d9a4c241e42f..14399cc02b22 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt @@ -6,7 +6,7 @@ barretenberg_module( stdlib_blake2s stdlib_keccak stdlib_pedersen_hash - stdlib_merkle_tree + crypto_merkle_tree stdlib_schnorr crypto_sha256 ) diff --git a/barretenberg/cpp/src/barretenberg/dsl/types.hpp b/barretenberg/cpp/src/barretenberg/dsl/types.hpp index b5bf296cbf53..8dd4a55ac9ff 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/types.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/types.hpp @@ -1,11 +1,11 @@ #pragma once #include "barretenberg/plonk/composer/ultra_composer.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/goblin/goblin.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp" #include "barretenberg/stdlib/primitives/biggroup/biggroup.hpp" #include "barretenberg/stdlib/primitives/bit_array/bit_array.hpp" @@ -58,7 +58,7 @@ using bn254 = bb::stdlib::bn254; using secp256k1_ct = bb::stdlib::secp256k1; using secp256r1_ct = bb::stdlib::secp256r1; -using hash_path_ct = bb::stdlib::merkle_tree::hash_path; +using hash_path_ct = bb::crypto::merkle_tree::hash_path; using schnorr_signature_bits_ct = bb::stdlib::schnorr_signature_bits; diff --git a/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt index ee4b04affe82..60ba6aa607c3 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(goblin stdlib_recursion ultra_honk eccvm translator_vm stdlib_sha256 stdlib_merkle_tree stdlib_primitives) \ No newline at end of file +barretenberg_module(goblin stdlib_recursion ultra_honk eccvm translator_vm stdlib_sha256 crypto_merkle_tree stdlib_primitives) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp index 12fde5fb31b6..74efb83c615d 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp @@ -2,15 +2,15 @@ #include "barretenberg/commitment_schemes/commitment_key.hpp" #include "barretenberg/crypto/ecdsa/ecdsa.hpp" +#include "barretenberg/crypto/merkle_tree/membership.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/flavor/goblin_ultra.hpp" #include "barretenberg/goblin/goblin.hpp" #include "barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp" #include "barretenberg/srs/global_crs.hpp" #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" #include "barretenberg/stdlib/hash/sha256/sha256.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" #include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp" #include "barretenberg/stdlib/recursion/honk/verifier/protogalaxy_recursive_verifier.hpp" diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt index 2a44f93e256e..9c90ad2ba844 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt @@ -8,4 +8,4 @@ barretenberg_module( stdlib_pedersen_commitment stdlib_schnorr stdlib_primitives - stdlib_merkle_tree) \ No newline at end of file + crypto_merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp index da7f5037d003..7d8533dd600f 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp @@ -1,7 +1,7 @@ #include "compute_circuit_data.hpp" #include "../notes/native/index.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "join_split_circuit.hpp" #include "sign_join_split_tx.hpp" @@ -10,7 +10,7 @@ namespace bb::join_split_example::proofs::join_split { using namespace bb::join_split_example::proofs::join_split; using namespace bb::stdlib; using namespace bb::join_split_example::proofs::notes::native; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; join_split_tx noop_tx() { diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp index 5eae28bf8de1..73cae41dbee9 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp @@ -7,7 +7,7 @@ namespace bb::join_split_example::proofs::join_split { using namespace bb::plonk; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; static std::shared_ptr proving_key; static std::shared_ptr verification_key; diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp index 5d110351dbb8..aa530c87a276 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp @@ -5,15 +5,15 @@ #include "../notes/native/index.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/crypto/merkle_tree/index.hpp" #include "barretenberg/join_split_example/types.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" -#include "barretenberg/stdlib/merkle_tree/index.hpp" #include "index.hpp" #include "join_split_circuit.hpp" namespace bb::join_split_example::proofs::join_split { -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; /* Old join-split tests below. The value of having all of these logic tests is unclear, but we'll leave them around, at least for a while. */ @@ -27,7 +27,7 @@ constexpr bool CIRCUIT_CHANGE_EXPECTED = false; using namespace bb; using namespace bb::stdlib; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::join_split_example::proofs::notes::native; using key_pair = join_split_example::fixtures::grumpkin_key_pair; @@ -53,7 +53,7 @@ class join_split_tests : public ::testing::Test { virtual void SetUp() { store = std::make_unique(); - tree = std::make_unique>(*store, 32); + tree = std::make_unique>(*store, 32); user = join_split_example::fixtures::create_user_context(); default_value_note = { .value = 100, @@ -302,7 +302,7 @@ class join_split_tests : public ::testing::Test { join_split_example::fixtures::user_context user; std::unique_ptr store; - std::unique_ptr> tree; + std::unique_ptr> tree; bridge_call_data empty_bridge_call_data = { .bridge_address_id = 0, .input_asset_id_a = 0, .input_asset_id_b = 0, diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp index 847e7d15cff7..109deccf8294 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp @@ -4,15 +4,15 @@ #include "../notes/circuit/claim/claim_note.hpp" #include "../notes/circuit/value/compute_nullifier.hpp" #include "../notes/circuit/value/value_note.hpp" +#include "barretenberg/crypto/merkle_tree/membership.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" #include "verify_signature.hpp" namespace bb::join_split_example::proofs::join_split { using namespace bb::plonk; using namespace notes::circuit; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::crypto; /** @@ -31,7 +31,7 @@ field_ct process_input_note(field_ct const& account_private_key, const bool_ct valid_value = note.value == 0 || is_note_in_use; valid_value.assert_equal(true, "padding note non zero"); - const bool_ct exists = bb::stdlib::merkle_tree::check_membership( + const bool_ct exists = bb::crypto::merkle_tree::check_membership( merkle_root, hash_path, note.commitment, index.value.decompose_into_bits(DATA_TREE_DEPTH)); const bool_ct valid = exists || is_propagated || !is_note_in_use; valid.assert_equal(true, "input note not a member"); @@ -217,7 +217,7 @@ join_split_outputs join_split_circuit_component(join_split_inputs const& inputs) const auto account_alias_hash = inputs.alias_hash; const auto account_note_data = account::account_note(account_alias_hash.value, account_public_key, signer); const bool_ct signing_key_exists = - stdlib::merkle_tree::check_membership(inputs.merkle_root, + crypto::merkle_tree::check_membership(inputs.merkle_root, inputs.account_note_path, account_note_data.commitment, inputs.account_note_index.value.decompose_into_bits(DATA_TREE_DEPTH)); @@ -288,8 +288,8 @@ void join_split_circuit(Builder& builder, join_split_tx const& tx) .signing_pub_key = group_ct::from_witness(&builder, tx.signing_pub_key), .signature = stdlib::schnorr_convert_signature(&builder, tx.signature), .merkle_root = witness_ct(&builder, tx.old_data_root), - .input_path1 = stdlib::merkle_tree::create_witness_hash_path(builder, tx.input_path[0]), - .input_path2 = stdlib::merkle_tree::create_witness_hash_path(builder, tx.input_path[1]), + .input_path1 = crypto::merkle_tree::create_witness_hash_path(builder, tx.input_path[0]), + .input_path2 = crypto::merkle_tree::create_witness_hash_path(builder, tx.input_path[1]), .account_note_index = suint_ct(witness_ct(&builder, tx.account_note_index), DATA_TREE_DEPTH, "account_note_index"), .account_note_path = merkle_tree::create_witness_hash_path(builder, tx.account_note_path), diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp index 6754c48cbd0d..ca3cbf3e2cb9 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp @@ -3,16 +3,16 @@ #include "../notes/native/index.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/crypto/merkle_tree/index.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" -#include "barretenberg/stdlib/merkle_tree/index.hpp" #include "index.hpp" namespace bb::join_split_example::proofs::join_split { using namespace bb; // using namespace bb::stdlib::types; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::join_split_example::proofs::notes::native; using key_pair = join_split_example::fixtures::grumpkin_key_pair; @@ -32,7 +32,7 @@ class join_split_js_parity_tests : public ::testing::Test { virtual void SetUp() { store = std::make_unique(); - tree = std::make_unique>(*store, 32); + tree = std::make_unique>(*store, 32); } void append_notes(std::vector const& notes) @@ -56,7 +56,7 @@ class join_split_js_parity_tests : public ::testing::Test { } std::unique_ptr store; - std::unique_ptr> tree; + std::unique_ptr> tree; }; TEST_F(join_split_js_parity_tests, test_full_proof) diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp index eb1282f00065..1e164be26877 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp @@ -1,9 +1,9 @@ #pragma once #include "../notes/native/claim/claim_note_tx_data.hpp" #include "../notes/native/value/value_note.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/schnorr/schnorr.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" namespace bb::join_split_example::proofs::join_split { @@ -15,7 +15,7 @@ struct join_split_tx { uint32_t num_input_notes; std::array input_index; bb::fr old_data_root; - std::array input_path; + std::array input_path; std::array input_note; std::array output_note; @@ -25,7 +25,7 @@ struct join_split_tx { bb::fr alias_hash; bool account_required; uint32_t account_note_index; - bb::stdlib::merkle_tree::fr_hash_path account_note_path; + bb::crypto::merkle_tree::fr_hash_path account_note_path; grumpkin::g1::affine_element signing_pub_key; bb::fr backward_link; // 0: no link, otherwise: any commitment. diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt index 0efb8cfc2750..ad23167bf652 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt @@ -6,4 +6,4 @@ barretenberg_module( stdlib_pedersen_commitment stdlib_schnorr stdlib_primitives - stdlib_merkle_tree) \ No newline at end of file + crypto_merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp b/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp index 68b0e79363d7..fbc2e45f7ca3 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp @@ -4,10 +4,10 @@ #include "barretenberg/plonk/composer/ultra_composer.hpp" #include "barretenberg/ultra_honk/ultra_composer.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bool/bool.hpp" #include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp" #include "barretenberg/stdlib/primitives/curves/bn254.hpp" @@ -37,7 +37,7 @@ using pedersen_commitment = bb::stdlib::pedersen_commitment; using pedersen_hash = bb::stdlib::pedersen_hash; using bn254 = bb::stdlib::bn254; -using hash_path_ct = bb::stdlib::merkle_tree::hash_path; +using hash_path_ct = bb::crypto::merkle_tree::hash_path; using schnorr_signature_bits = bb::stdlib::schnorr_signature_bits; diff --git a/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt index c3fa01c8518c..dea2efa6ea36 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt @@ -2,5 +2,4 @@ add_subdirectory(hash) add_subdirectory(commitment) add_subdirectory(encryption) add_subdirectory(primitives) -add_subdirectory(recursion) -add_subdirectory(merkle_tree) \ No newline at end of file +add_subdirectory(recursion) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp b/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp index d0658eef7095..6258d855e940 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp @@ -104,6 +104,18 @@ void native_pedersen_eight_hash_bench(State& state) noexcept } BENCHMARK(native_pedersen_eight_hash_bench)->MinTime(3); +void native_pedersen_hash_pair_bench(State& state) noexcept +{ + std::vector elements(2); + for (size_t i = 0; i < 2; ++i) { + elements[i] = grumpkin::fq::random_element(); + } + for (auto _ : state) { + crypto::pedersen_hash::hash(elements); + } +} +BENCHMARK(native_pedersen_hash_pair_bench)->Unit(benchmark::kMillisecond)->MinTime(3); + void construct_pedersen_witnesses_bench(State& state) noexcept { bb::srs::init_crs_factory(BARRETENBERG_SRS_PATH); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt deleted file mode 100644 index 0553ea7c3a62..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(stdlib_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp new file mode 100644 index 000000000000..d1dc13d95e52 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -0,0 +1,331 @@ +#include "indexed_tree.hpp" +#include "../array_store.hpp" +#include "../hash.hpp" +#include "../nullifier_tree/nullifier_memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "leaves_cache.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = Poseidon2HashPolicy; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +TEST(stdlib_indexed_tree, can_create) +{ + ArrayStore store(10); + IndexedTree tree = IndexedTree(store, 10); + EXPECT_EQ(tree.size(), 1ULL); + + NullifierMemoryTree memdb(10); + EXPECT_EQ(memdb.root(), tree.root()); +} + +TEST(stdlib_indexed_tree, test_size) +{ + ArrayStore store(32); + auto db = IndexedTree(store, 32); + + // We assume that the first leaf is already filled with (0, 0, 0). + EXPECT_EQ(db.size(), 1ULL); + + // Add a new non-zero leaf at index 1. + db.add_value(30); + EXPECT_EQ(db.size(), 2ULL); + + // Add third. + db.add_value(10); + EXPECT_EQ(db.size(), 3ULL); + + // Add forth. + db.add_value(20); + EXPECT_EQ(db.size(), 4ULL); +} + +TEST(stdlib_indexed_tree, test_get_hash_path) +{ + NullifierMemoryTree memdb(10); + + ArrayStore store(10); + auto db = IndexedTree(store, 10); + + EXPECT_EQ(memdb.root(), db.root()); + + EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); + + memdb.update_element(VALUES[512]); + db.add_value(VALUES[512]); + + EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); + + for (size_t i = 0; i < 512; ++i) { + memdb.update_element(VALUES[i]); + db.add_value(VALUES[i]); + } + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); +} + +TEST(stdlib_indexed_tree, test_batch_insert) +{ + const size_t batch_size = 16; + const size_t num_batches = 16; + size_t depth = 10; + NullifierMemoryTree memdb(depth, batch_size); + + ArrayStore store1(depth); + IndexedTree tree1 = + IndexedTree(store1, depth, batch_size); + + ArrayStore store2(depth); + IndexedTree tree2 = + IndexedTree(store2, depth, batch_size); + + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t i = 0; i < num_batches; i++) { + std::vector batch; + std::vector memory_tree_hash_paths; + for (size_t j = 0; j < batch_size; j++) { + batch.push_back(fr(random_engine.get_random_uint256())); + fr_hash_path path = memdb.update_element(batch[j]); + memory_tree_hash_paths.push_back(path); + } + std::vector tree1_hash_paths = tree1.add_or_update_values(batch, true); + std::vector tree2_hash_paths = tree2.add_or_update_values(batch); + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t j = 0; j < batch_size; j++) { + EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); + // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); + } + } +} + +fr hash_leaf(const indexed_leaf& leaf) +{ + return HashPolicy::hash(leaf.get_hash_inputs()); +} + +bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const size_t idx) +{ + auto current = hash_leaf(leaf_value); + size_t depth_ = path.size(); + size_t index = idx; + for (size_t i = 0; i < depth_; ++i) { + fr left = (index & 1) ? path[i].first : current; + fr right = (index & 1) ? current : path[i].second; + current = HashPolicy::hash_pair(left, right); + index >>= 1; + } + return current == root; +} + +TEST(stdlib_indexed_tree, test_indexed_memory) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(tree.get_leaf(0), zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.add_value(30); + EXPECT_EQ(tree.size(), 2); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.add_value(10); + EXPECT_EQ(tree.size(), 3); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 1, 30 })); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.add_value(20); + EXPECT_EQ(tree.size(), 4); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + + // Adding the same value must not affect anything + // tree.update_element(20); + // EXPECT_EQ(tree.get_leaves().size(), 4); + // EXPECT_EQ(tree.get_leaves()[0], hash_leaf({ 0, 2, 10 })); + // EXPECT_EQ(tree.get_leaves()[1], hash_leaf({ 30, 0, 0 })); + // EXPECT_EQ(tree.get_leaves()[2], hash_leaf({ 10, 3, 20 })); + // EXPECT_EQ(tree.get_leaves()[3], hash_leaf({ 20, 1, 30 })); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.add_value(50); + EXPECT_EQ(tree.size(), 5); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(4)), hash_leaf({ 50, 0, 0 })); + + // Manually compute the node values + auto e000 = hash_leaf(tree.get_leaf(0)); + auto e001 = hash_leaf(tree.get_leaf(1)); + auto e010 = hash_leaf(tree.get_leaf(2)); + auto e011 = hash_leaf(tree.get_leaf(3)); + auto e100 = hash_leaf(tree.get_leaf(4)); + auto e101 = hash_leaf({ 0, 0, 0 }); + auto e110 = hash_leaf({ 0, 0, 0 }); + auto e111 = hash_leaf({ 0, 0, 0 }); + + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected); + EXPECT_EQ(tree.get_hash_path(1), expected); + fr_hash_path expected2 = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected2); + EXPECT_EQ(tree.get_hash_path(3), expected2); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} + +TEST(stdlib_indexed_tree, test_indexed_tree) +{ + // Create a depth-8 indexed merkle tree + constexpr size_t depth = 8; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); + + // Add 20 random values to the tree + for (size_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + tree.add_value(value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + std::vector differences; + for (size_t i = 0; i < size_t(tree.size()); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + auto hash_path = tree.get_hash_path(index); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp deleted file mode 100644 index d15cd5976acc..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include "memory_tree.hpp" -#include "hash.hpp" - -namespace bb::stdlib::merkle_tree { - -MemoryTree::MemoryTree(size_t depth) - : depth_(depth) -{ - - ASSERT(depth_ >= 1 && depth <= 20); - total_size_ = 1UL << depth_; - hashes_.resize(total_size_ * 2 - 2); - - // Build the entire tree. - auto current = fr(0); - size_t layer_size = total_size_; - for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { - for (size_t i = 0; i < layer_size; ++i) { - hashes_[offset + i] = current; - } - current = hash_pair_native(current, current); - } - - root_ = current; -} - -fr_hash_path MemoryTree::get_hash_path(size_t index) -{ - fr_hash_path path(depth_); - size_t offset = 0; - size_t layer_size = total_size_; - for (size_t i = 0; i < depth_; ++i) { - index -= index & 0x1; - path[i] = std::make_pair(hashes_[offset + index], hashes_[offset + index + 1]); - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - return path; -} - -fr_sibling_path MemoryTree::get_sibling_path(size_t index) -{ - fr_sibling_path path(depth_); - size_t offset = 0; - size_t layer_size = total_size_; - for (size_t i = 0; i < depth_; i++) { - if (index % 2 == 0) { - path[i] = hashes_[offset + index + 1]; - } else { - path[i] = hashes_[offset + index - 1]; - } - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - return path; -} - -fr MemoryTree::update_element(size_t index, fr const& value) -{ - size_t offset = 0; - size_t layer_size = total_size_; - fr current = value; - for (size_t i = 0; i < depth_; ++i) { - hashes_[offset + index] = current; - index &= (~0ULL) - 1; - current = hash_pair_native(hashes_[offset + index], hashes_[offset + index + 1]); - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - root_ = current; - return root_; -} - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp deleted file mode 100644 index d16e6388c485..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include "hash_path.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -/** - * A MemoryTree is structured as follows: - * hashes_ - * +------------------------------------------------------------------------------+ - * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | - * i | | - * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | - * d | | - * e | 12 -> h_{2,0} h_{2,1} | - * x | | - * | 14 -> h_{3,0} | - * +------------------------------------------------------------------------------+ - * - * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. - * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. - * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. - */ -class MemoryTree { - public: - MemoryTree(size_t depth); - - fr_hash_path get_hash_path(size_t index); - - fr_sibling_path get_sibling_path(size_t index); - - fr update_element(size_t index, fr const& value); - - fr root() const { return root_; } - - public: - size_t depth_; - size_t total_size_; - bb::fr root_; - std::vector hashes_; -}; - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp deleted file mode 100644 index 86e973a18f60..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "hash_path.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -class MemoryStore; - -template class MerkleTree { - public: - typedef uint256_t index_t; - - MerkleTree(Store& store, size_t depth, uint8_t tree_id = 0); - MerkleTree(MerkleTree const& other) = delete; - MerkleTree(MerkleTree&& other); - ~MerkleTree(); - - fr_hash_path get_hash_path(index_t index); - - fr_sibling_path get_sibling_path(index_t index); - - fr update_element(index_t index, fr const& value); - - fr root() const; - - size_t depth() const { return depth_; } - - index_t size() const; - - protected: - void load_metadata(); - - /** - * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. - * - * @param height: The tree depth - * @param index: the index of the non-empty leaf - * @param value: the value to be stored in the non-empty leaf - * - * @see Check full documentation: https://hackmd.io/2zyJc6QhRuugyH8D78Tbqg?view - */ - fr update_element(fr const& root, fr const& value, index_t index, size_t height); - - fr get_element(fr const& root, index_t index, size_t height); - - /** - * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. - * - * @param height: The tree depth - * @param index: the index of the non-empty leaf - * @param value: the value to be stored in the non-empty leaf - */ - fr compute_zero_path_hash(size_t height, index_t index, fr const& value); - - /** - * Given child nodes `a` and `b` and index of `a`, compute their parent node `p` and store [p : (a, b)]. - * - * @param a_index: the index of the child node `a` - * @param a: child node - * @param b: child node - * @param height: the height of the parent node - */ - fr binary_put(index_t a_index, fr const& a, fr const& b, size_t height); - - fr fork_stump( - fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t stump_height); - - /** - * Stores a parent node and child nodes in the database as [key : (left, right)]. - * - * @param key: The node value to be stored as key - * @param left: the left child node - * @param right: the right child node - */ - void put(fr const& key, fr const& left, fr const& right); - - /** - * Stores a stump [key : (value, index, true)] in the memory. - * The additional byte `true` is to denote this is a stump. - * - * @param key: The node value to be stored as key - * @param value: value of the non-empty leaf in the stump - * @param index: the index of the non-empty leaf in the stump - */ - void put_stump(fr const& key, index_t index, fr const& value); - - void remove(fr const& key); - - protected: - Store& store_; - std::vector zero_hashes_; - size_t depth_; - uint8_t tree_id_; -}; - -} // namespace bb::stdlib::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp deleted file mode 100644 index df0acdccefc8..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "nullifier_memory_tree.hpp" -#include "../hash.hpp" - -namespace bb::stdlib::merkle_tree { - -NullifierMemoryTree::NullifierMemoryTree(size_t depth) - : MemoryTree(depth) -{ - ASSERT(depth_ >= 1 && depth <= 32); - total_size_ = 1UL << depth_; - hashes_.resize(total_size_ * 2 - 2); - - // Build the entire tree and fill with 0 hashes. - auto current = WrappedNullifierLeaf::zero().hash(); - size_t layer_size = total_size_; - for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { - for (size_t i = 0; i < layer_size; ++i) { - hashes_[offset + i] = current; - } - current = hash_pair_native(current, current); - } - - // Insert the initial leaf at index 0 - auto initial_leaf = WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); - leaves_.push_back(initial_leaf); - root_ = update_element(0, initial_leaf.hash()); -} - -fr NullifierMemoryTree::update_element(fr const& value) -{ - // Find the leaf with the value closest and less than `value` - - // If value is 0 we simply append 0 a null NullifierLeaf to the tree - if (value == 0) { - auto zero_leaf = WrappedNullifierLeaf::zero(); - leaves_.push_back(zero_leaf); - return update_element(leaves_.size() - 1, zero_leaf.hash()); - } - - size_t current; - bool is_already_present; - std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); - - nullifier_leaf current_leaf = leaves_[current].unwrap(); - nullifier_leaf new_leaf = { .value = value, - .nextIndex = current_leaf.nextIndex, - .nextValue = current_leaf.nextValue }; - - if (!is_already_present) { - // Update the current leaf to point it to the new leaf - current_leaf.nextIndex = leaves_.size(); - current_leaf.nextValue = value; - - leaves_[current].set(current_leaf); - - // Insert the new leaf with (nextIndex, nextValue) of the current leaf - leaves_.push_back(new_leaf); - } - - // Update the old leaf in the tree - auto old_leaf_hash = current_leaf.hash(); - size_t old_leaf_index = current; - auto root = update_element(old_leaf_index, old_leaf_hash); - - // Insert the new leaf in the tree - auto new_leaf_hash = new_leaf.hash(); - size_t new_leaf_index = is_already_present ? old_leaf_index : leaves_.size() - 1; - root = update_element(new_leaf_index, new_leaf_hash); - - return root; -} - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp deleted file mode 100644 index 624daba7ad9c..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once -#include "../hash.hpp" -#include "../memory_tree.hpp" -#include "nullifier_leaf.hpp" - -namespace bb::stdlib::merkle_tree { - -/** - * An NullifierMemoryTree is structured just like a usual merkle tree: - * - * hashes_ - * +------------------------------------------------------------------------------+ - * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | - * i | | - * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | - * d | | - * e | 12 -> h_{2,0} h_{2,1} | - * x | | - * | 14 -> h_{3,0} | - * +------------------------------------------------------------------------------+ - * - * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. - * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. - * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. - * - * 1. Initial state: - * - * # - * - * # # - * - * # # # # - * - * # # # # # # # # - * - * index 0 1 2 3 4 5 6 7 - * - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextVal 0 0 0 0 0 0 0 0 - * - * 2. Add new leaf with value 30 - * - * val 0 30 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextVal 30 0 0 0 0 0 0 0 - * - * 3. Add new leaf with value 10 - * - * val 0 30 10 0 0 0 0 0 - * nextIdx 2 0 1 0 0 0 0 0 - * nextVal 10 0 30 0 0 0 0 0 - * - * 4. Add new leaf with value 20 - * - * val 0 30 10 20 0 0 0 0 - * nextIdx 2 0 3 1 0 0 0 0 - * nextVal 10 0 20 30 0 0 0 0 - * - * 5. Add new leaf with value 50 - * - * val 0 30 10 20 50 0 0 0 - * nextIdx 2 4 3 1 0 0 0 0 - * nextVal 10 50 20 30 0 0 0 0 - */ -class NullifierMemoryTree : public MemoryTree { - - public: - NullifierMemoryTree(size_t depth); - - using MemoryTree::get_hash_path; - using MemoryTree::root; - using MemoryTree::update_element; - - fr update_element(fr const& value); - - const std::vector& get_hashes() { return hashes_; } - const WrappedNullifierLeaf get_leaf(size_t index) - { - return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf::zero(); - } - const std::vector& get_leaves() { return leaves_; } - - protected: - using MemoryTree::depth_; - using MemoryTree::hashes_; - using MemoryTree::root_; - using MemoryTree::total_size_; - std::vector leaves_; -}; - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp deleted file mode 100644 index 2135f34da3a9..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once -#include "../hash.hpp" -#include "../merkle_tree.hpp" -#include "nullifier_leaf.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -template class NullifierTree : public MerkleTree { - public: - typedef uint256_t index_t; - - NullifierTree(Store& store, size_t depth, uint8_t tree_id = 0); - NullifierTree(NullifierTree const& other) = delete; - NullifierTree(NullifierTree&& other); - ~NullifierTree(); - - using MerkleTree::get_hash_path; - using MerkleTree::root; - using MerkleTree::size; - using MerkleTree::depth; - - fr update_element(fr const& value); - - private: - using MerkleTree::update_element; - using MerkleTree::get_element; - using MerkleTree::compute_zero_path_hash; - - private: - using MerkleTree::store_; - using MerkleTree::zero_hashes_; - using MerkleTree::depth_; - using MerkleTree::tree_id_; - std::vector leaves; -}; - -} // namespace bb::stdlib::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp b/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp index 96bf0d2b2033..e97d63f0e4ba 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp @@ -1,11 +1,11 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/plonk/proof_system/types/prover_settings.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp" #include "barretenberg/stdlib/primitives/biggroup/biggroup.hpp" #include "barretenberg/stdlib/primitives/bit_array/bit_array.hpp" @@ -52,11 +52,6 @@ using cycle_group_ct = stdlib::cycle_group; using bn254 = stdlib::bn254; using secp256k1_ct = stdlib::secp256k1; -namespace merkle_tree { -using namespace stdlib::merkle_tree; -using hash_path = stdlib::merkle_tree::hash_path; -} // namespace merkle_tree - using schnorr_signature_bits = stdlib::schnorr_signature_bits; // Ultra-composer specific types diff --git a/barretenberg/exports.json b/barretenberg/exports.json index f04b2871163d..13425b898cda 100644 --- a/barretenberg/exports.json +++ b/barretenberg/exports.json @@ -35,6 +35,26 @@ ], "isAsync": false }, + { + "functionName": "pedersen_hashes", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + }, + { + "name": "hash_index", + "type": "const uint32_t *" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, { "functionName": "pedersen_hash_buffer", "inArgs": [ @@ -55,6 +75,38 @@ ], "isAsync": false }, + { + "functionName": "poseidon_hash", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, + { + "functionName": "poseidon_hashes", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, { "functionName": "blake2s", "inArgs": [ diff --git a/barretenberg/scripts/bindgen.sh b/barretenberg/scripts/bindgen.sh index e3080a848242..e3d5b9b5a70c 100755 --- a/barretenberg/scripts/bindgen.sh +++ b/barretenberg/scripts/bindgen.sh @@ -3,4 +3,4 @@ set -eu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json -(cd ./ts && yarn ts-node-esm ./src/bindgen/index.ts ../exports.json > ./src/barretenberg_api/index.ts) +(cd ./ts && yarn node --loader ts-node/esm ./src/bindgen/index.ts ../exports.json > ./src/barretenberg_api/index.ts) \ No newline at end of file diff --git a/barretenberg/scripts/c_bind_files.txt b/barretenberg/scripts/c_bind_files.txt index 255fcd4f5ad6..5b6f0ffc5d0b 100644 --- a/barretenberg/scripts/c_bind_files.txt +++ b/barretenberg/scripts/c_bind_files.txt @@ -1,5 +1,6 @@ ./cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp ./cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +./cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp ./cpp/src/barretenberg/crypto/blake2s/c_bind.hpp ./cpp/src/barretenberg/crypto/schnorr/c_bind.hpp ./cpp/src/barretenberg/crypto/aes128/c_bind.hpp diff --git a/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap b/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap new file mode 100644 index 000000000000..3af7544b2b80 --- /dev/null +++ b/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap @@ -0,0 +1,40 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`poseidon sync poseidonHash 1`] = ` +Fr { + "value": Uint8Array [ + 43, + 202, + 235, + 109, + 88, + 187, + 56, + 186, + 247, + 83, + 213, + 140, + 60, + 150, + 97, + 143, + 234, + 130, + 22, + 51, + 69, + 41, + 94, + 180, + 14, + 136, + 52, + 78, + 235, + 12, + 226, + 161, + ], +} +`; diff --git a/barretenberg/ts/src/barretenberg/pedersen.test.ts b/barretenberg/ts/src/barretenberg/pedersen.test.ts index 4b0150ab4db8..42ba13f9c8ba 100644 --- a/barretenberg/ts/src/barretenberg/pedersen.test.ts +++ b/barretenberg/ts/src/barretenberg/pedersen.test.ts @@ -14,6 +14,29 @@ describe('pedersen sync', () => { expect(result).toMatchSnapshot(); }); + it('pedersenHash perf test', () => { + const loops = 1000; + const fields = Array.from({ length: loops * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.pedersenHash([fields[i * 2], fields[i * 2 + 1]], 0); + } + const us = t.us() / loops; + console.log(`Executed ${loops} hashes at an average ${us}us / hash`); + }); + + it('pedersenHashes perf test', () => { + const loops = 10; + const numHashesPerLoop = 1024; + const fields = Array.from({ length: numHashesPerLoop * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.pedersenHashes(fields, 0); + } + const us = t.us() / (numHashesPerLoop * loops); + console.log(`Executed ${numHashesPerLoop * loops} hashes at an average ${us}us / hash`); + }); + it('pedersenHashBuffer', () => { const input = Buffer.alloc(123); input.writeUint32BE(321, 0); diff --git a/barretenberg/ts/src/barretenberg/poseidon.test.ts b/barretenberg/ts/src/barretenberg/poseidon.test.ts new file mode 100644 index 000000000000..2647e4778ddc --- /dev/null +++ b/barretenberg/ts/src/barretenberg/poseidon.test.ts @@ -0,0 +1,39 @@ +import { BarretenbergSync } from './index.js'; +import { Timer } from '../benchmark/timer.js'; +import { Fr } from '../types/index.js'; + +describe('poseidon sync', () => { + let api: BarretenbergSync; + + beforeAll(async () => { + api = await BarretenbergSync.new(); + }); + + it('poseidonHash', () => { + const result = api.poseidonHash([new Fr(4n), new Fr(8n)]); + expect(result).toMatchSnapshot(); + }); + + it('poseidonHash perf test', () => { + const loops = 1000; + const fields = Array.from({ length: loops * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.poseidonHash([fields[i * 2], fields[i * 2 + 1]]); + } + const us = t.us() / loops; + console.log(`Executed ${loops} hashes at an average ${us}us / hash`); + }); + + it('poseidonHashes perf test', () => { + const loops = 10; + const numHashesPerLoop = 1024; + const fields = Array.from({ length: numHashesPerLoop * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.poseidonHashes(fields); + } + const us = t.us() / (numHashesPerLoop * loops); + console.log(`Executed ${numHashesPerLoop * loops} hashes at an average ${us}us / hash`); + }); +}); diff --git a/barretenberg/ts/src/barretenberg_api/index.ts b/barretenberg/ts/src/barretenberg_api/index.ts index 1a337c33e2ac..f74134d54675 100644 --- a/barretenberg/ts/src/barretenberg_api/index.ts +++ b/barretenberg/ts/src/barretenberg_api/index.ts @@ -39,6 +39,18 @@ export class BarretenbergApi { return out[0]; } + async pedersenHashes(inputsBuffer: Fr[], hashIndex: number): Promise { + const inArgs = [inputsBuffer, hashIndex].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'pedersen_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async pedersenHashBuffer(inputBuffer: Uint8Array, hashIndex: number): Promise { const inArgs = [inputBuffer, hashIndex].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; @@ -51,6 +63,30 @@ export class BarretenbergApi { return out[0]; } + async poseidonHash(inputsBuffer: Fr[]): Promise { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'poseidon_hash', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + + async poseidonHashes(inputsBuffer: Fr[]): Promise { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'poseidon_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async blake2s(data: Uint8Array): Promise { const inArgs = [data].map(serializeBufferable); const outTypes: OutputType[] = [Buffer32]; @@ -551,6 +587,18 @@ export class BarretenbergApiSync { return out[0]; } + pedersenHashes(inputsBuffer: Fr[], hashIndex: number): Fr { + const inArgs = [inputsBuffer, hashIndex].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'pedersen_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + pedersenHashBuffer(inputBuffer: Uint8Array, hashIndex: number): Fr { const inArgs = [inputBuffer, hashIndex].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; @@ -563,6 +611,30 @@ export class BarretenbergApiSync { return out[0]; } + poseidonHash(inputsBuffer: Fr[]): Fr { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'poseidon_hash', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + + poseidonHashes(inputsBuffer: Fr[]): Fr { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'poseidon_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + blake2s(data: Uint8Array): Buffer32 { const inArgs = [data].map(serializeBufferable); const outTypes: OutputType[] = [Buffer32]; From 79bf44581ea3039a33ea63bb1a2ed429bfa0ece8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Fri, 9 Feb 2024 16:31:13 +0100 Subject: [PATCH 2/3] refactor: removing redundant utilities (#4532) Fixes #3470 --- .../src/history/nullifier_non_inclusion.nr | 13 +- .../src/history/public_value_inclusion.nr | 6 +- yarn-project/aztec-nr/aztec/src/lib.nr | 1 - .../messaging/l1_to_l2_message_getter_data.nr | 10 +- yarn-project/aztec-nr/aztec/src/note/utils.nr | 2 +- .../src/oracle/get_membership_witness.nr | 12 +- .../get_nullifier_membership_witness.nr | 2 +- .../src/oracle/get_public_data_witness.nr | 30 +-- .../aztec-nr/aztec/src/oracle/notes.nr | 2 +- yarn-project/aztec-nr/aztec/src/utils.nr | 18 -- .../src/crates/rollup-lib/src/abis.nr | 1 - .../rollup-lib/src/base/base_rollup_inputs.nr | 213 +++++++++++------- .../src/crates/types/src/lib.nr | 2 + .../crates/types/src/public_data_tree_leaf.nr | 28 +++ .../src/public_data_tree_leaf_preimage.nr} | 29 +-- .../src/crates/types/src/utils.nr | 1 - .../src/crates/types/src/utils/field.nr | 9 + 17 files changed, 198 insertions(+), 181 deletions(-) delete mode 100644 yarn-project/aztec-nr/aztec/src/utils.nr create mode 100644 yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf.nr rename yarn-project/noir-protocol-circuits/src/crates/{rollup-lib/src/abis/public_data_tree_leaf.nr => types/src/public_data_tree_leaf_preimage.nr} (58%) diff --git a/yarn-project/aztec-nr/aztec/src/history/nullifier_non_inclusion.nr b/yarn-project/aztec-nr/aztec/src/history/nullifier_non_inclusion.nr index 5568321fbd25..d202ac3a290c 100644 --- a/yarn-project/aztec-nr/aztec/src/history/nullifier_non_inclusion.nr +++ b/yarn-project/aztec-nr/aztec/src/history/nullifier_non_inclusion.nr @@ -1,6 +1,11 @@ use dep::std::merkle::compute_merkle_root; -use dep::protocol_types::header::Header; - +use dep::protocol_types::{ + header::Header, + utils::field::{ + full_field_less_than, + full_field_greater_than, + }, +}; use crate::{ context::PrivateContext, note::{ @@ -8,10 +13,6 @@ use crate::{ note_interface::NoteInterface, }, oracle::get_nullifier_membership_witness::get_low_nullifier_membership_witness, - utils::{ - full_field_less_than, - full_field_greater_than, - }, }; pub fn _nullifier_non_inclusion(nullifier: Field, header: Header) { diff --git a/yarn-project/aztec-nr/aztec/src/history/public_value_inclusion.nr b/yarn-project/aztec-nr/aztec/src/history/public_value_inclusion.nr index fbbfe95b9a9c..5a5f7f13dc58 100644 --- a/yarn-project/aztec-nr/aztec/src/history/public_value_inclusion.nr +++ b/yarn-project/aztec-nr/aztec/src/history/public_value_inclusion.nr @@ -1,10 +1,11 @@ use dep::protocol_types::{ constants::GENERATOR_INDEX__PUBLIC_LEAF_INDEX, - header::Header, hash::pedersen_hash, address::{ AztecAddress }, + header::Header, + utils::field::full_field_less_than, }; use dep::std::merkle::compute_merkle_root; @@ -13,9 +14,6 @@ use crate::{ oracle::get_public_data_witness::{ get_public_data_witness, }, - utils::{ - full_field_less_than, - }, }; fn _public_value_inclusion( diff --git a/yarn-project/aztec-nr/aztec/src/lib.nr b/yarn-project/aztec-nr/aztec/src/lib.nr index 4b02952d76a1..2ca0b281a5af 100644 --- a/yarn-project/aztec-nr/aztec/src/lib.nr +++ b/yarn-project/aztec-nr/aztec/src/lib.nr @@ -9,5 +9,4 @@ mod messaging; mod note; mod oracle; mod state_vars; -mod utils; use dep::protocol_types; diff --git a/yarn-project/aztec-nr/aztec/src/messaging/l1_to_l2_message_getter_data.nr b/yarn-project/aztec-nr/aztec/src/messaging/l1_to_l2_message_getter_data.nr index 34f21c60395f..0f7254710c45 100644 --- a/yarn-project/aztec-nr/aztec/src/messaging/l1_to_l2_message_getter_data.nr +++ b/yarn-project/aztec-nr/aztec/src/messaging/l1_to_l2_message_getter_data.nr @@ -1,9 +1,11 @@ use crate::messaging::l1_to_l2_message::L1ToL2Message; -use dep::protocol_types::constants::{ - L1_TO_L2_MSG_TREE_HEIGHT, - L1_TO_L2_MESSAGE_LENGTH, +use dep::protocol_types::{ + constants::{ + L1_TO_L2_MSG_TREE_HEIGHT, + L1_TO_L2_MESSAGE_LENGTH, + }, + utils::arr_copy_slice, }; -use crate::utils::arr_copy_slice; struct L1ToL2MessageGetterData { message: L1ToL2Message, diff --git a/yarn-project/aztec-nr/aztec/src/note/utils.nr b/yarn-project/aztec-nr/aztec/src/note/utils.nr index 5c05cd1b7cad..81f2f2c17ce8 100644 --- a/yarn-project/aztec-nr/aztec/src/note/utils.nr +++ b/yarn-project/aztec-nr/aztec/src/note/utils.nr @@ -4,7 +4,6 @@ use crate::{ note_header::NoteHeader, note_interface::NoteInterface, }, - utils::arr_copy_slice, }; use dep::protocol_types::{ @@ -15,6 +14,7 @@ use dep::protocol_types::{ GENERATOR_INDEX__SILOED_COMMITMENT, }, hash::pedersen_hash, + utils::arr_copy_slice, }; fn compute_siloed_hash(contract_address: AztecAddress, inner_note_hash: Field) -> Field { diff --git a/yarn-project/aztec-nr/aztec/src/oracle/get_membership_witness.nr b/yarn-project/aztec-nr/aztec/src/oracle/get_membership_witness.nr index d0ae5e8608bd..14d5b9e20ff4 100644 --- a/yarn-project/aztec-nr/aztec/src/oracle/get_membership_witness.nr +++ b/yarn-project/aztec-nr/aztec/src/oracle/get_membership_witness.nr @@ -1,9 +1,11 @@ -use dep::protocol_types::constants::{ - ARCHIVE_HEIGHT, - CONTRACT_TREE_HEIGHT, - NOTE_HASH_TREE_HEIGHT, +use dep::protocol_types::{ + constants::{ + ARCHIVE_HEIGHT, + CONTRACT_TREE_HEIGHT, + NOTE_HASH_TREE_HEIGHT, + }, + utils::arr_copy_slice, }; -use crate::utils::arr_copy_slice; global CONTRACT_TREE_ID = 0; global NOTE_HASH_TREE_ID = 2; diff --git a/yarn-project/aztec-nr/aztec/src/oracle/get_nullifier_membership_witness.nr b/yarn-project/aztec-nr/aztec/src/oracle/get_nullifier_membership_witness.nr index 21138a3ecf1e..3cf667ce7157 100644 --- a/yarn-project/aztec-nr/aztec/src/oracle/get_nullifier_membership_witness.nr +++ b/yarn-project/aztec-nr/aztec/src/oracle/get_nullifier_membership_witness.nr @@ -5,8 +5,8 @@ use dep::protocol_types::{ }, constants::NULLIFIER_TREE_HEIGHT, hash::pedersen_hash, + utils::arr_copy_slice, }; -use crate::utils::arr_copy_slice; // INDEX_LENGTH + NULLIFIER_LEAF_PREIMAGE_LENGTH + NULLIFIER_TREE_HEIGHT global NULLIFIER_MEMBERSHIP_WITNESS: Field = 24; diff --git a/yarn-project/aztec-nr/aztec/src/oracle/get_public_data_witness.nr b/yarn-project/aztec-nr/aztec/src/oracle/get_public_data_witness.nr index f0c4ce2f8e9f..1e843f794080 100644 --- a/yarn-project/aztec-nr/aztec/src/oracle/get_public_data_witness.nr +++ b/yarn-project/aztec-nr/aztec/src/oracle/get_public_data_witness.nr @@ -1,35 +1,17 @@ use dep::protocol_types::{ constants::PUBLIC_DATA_TREE_HEIGHT, hash::pedersen_hash, - traits::{Hash, Serialize} + public_data_tree_leaf_preimage::PublicDataTreeLeafPreimage, + traits::{ + Hash, + Serialize, + }, + utils::arr_copy_slice, }; -use crate::utils::arr_copy_slice; global LEAF_PREIMAGE_LENGTH: Field = 4; -// TODO: move this to constants_gen.nr so that it gets computed as INDEX_LENGTH + LEAF_DATA_LENGTH + PUBLIC_DATA_TREE_HEIGHT global PUBLIC_DATA_WITNESS: Field = 45; -// TODO(#3470) replace with /mnt/user-data/jan/aztec-packages/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/public_data_tree_leaf.nr -struct PublicDataTreeLeafPreimage { - slot : Field, - value: Field, - next_index : u32, - next_slot :Field, -} - -impl Serialize for PublicDataTreeLeafPreimage { - fn serialize(self) -> [Field; LEAF_PREIMAGE_LENGTH] { - [self.slot, self.value, self.next_index as Field, self.next_slot] - } -} - -impl Hash for PublicDataTreeLeafPreimage { - fn hash(self) -> Field { - // Performs the same hashing as StandardIndexedTree::encodeLeaf(...) - pedersen_hash(self.serialize(), 0) - } -} - struct PublicDataWitness { index: Field, leaf_preimage: PublicDataTreeLeafPreimage, diff --git a/yarn-project/aztec-nr/aztec/src/oracle/notes.nr b/yarn-project/aztec-nr/aztec/src/oracle/notes.nr index 1c49f54f0c11..f6378c1ecd68 100644 --- a/yarn-project/aztec-nr/aztec/src/oracle/notes.nr +++ b/yarn-project/aztec-nr/aztec/src/oracle/notes.nr @@ -3,10 +3,10 @@ use crate::note::{ note_header::NoteHeader, note_interface::NoteInterface, }; -use crate::utils::arr_copy_slice; use dep::protocol_types::{ address::AztecAddress, + utils::arr_copy_slice, }; #[oracle(notifyCreatedNote)] diff --git a/yarn-project/aztec-nr/aztec/src/utils.nr b/yarn-project/aztec-nr/aztec/src/utils.nr deleted file mode 100644 index 9d66e257b9c5..000000000000 --- a/yarn-project/aztec-nr/aztec/src/utils.nr +++ /dev/null @@ -1,18 +0,0 @@ -pub fn arr_copy_slice(src: [T; N], mut dst: [T; M], offset: Field) -> [T; M] { - for i in 0..dst.len() { - dst[i] = src[i + offset]; - } - dst -} - -// TODO(#3470): Copied over from https://github.com/AztecProtocol/aztec-packages/blob/a07c4bd47313be6aa604a63f37857eb0136b41ba/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr#L599 -// move to a shared place? - -// TODO to radix returns u8, so we cannot use bigger radixes. It'd be ideal to use a radix of the maximum range-constrained integer noir supports -pub fn full_field_less_than(lhs: Field, rhs: Field) -> bool { - lhs.lt(rhs) -} - -pub fn full_field_greater_than(lhs: Field, rhs: Field) -> bool { - rhs.lt(lhs) -} diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis.nr index 980149dab8a5..fe5d8000c098 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis.nr @@ -1,4 +1,3 @@ -mod public_data_tree_leaf; mod constant_rollup_data; mod base_or_merge_rollup_public_inputs; mod previous_rollup_data; diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr index 3181b8cdbd2c..73099641c563 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr @@ -1,41 +1,72 @@ -use crate::abis::public_data_tree_leaf::{PublicDataTreeLeaf, PublicDataTreeLeafPreimage}; -use crate::abis::constant_rollup_data::ConstantRollupData; -use crate::abis::base_or_merge_rollup_public_inputs::{BaseOrMergeRollupPublicInputs, BASE_ROLLUP_TYPE}; -use crate::base::state_diff_hints::StateDiffHints; -use crate::merkle_tree::{calculate_subtree, calculate_empty_tree_root}; -use crate::components; -use dep::types::utils::uint256::U256; -use dep::types::abis::append_only_tree_snapshot::AppendOnlyTreeSnapshot; -use dep::types::abis::public_data_update_request::PublicDataUpdateRequest; -use dep::types::abis::public_data_read::PublicDataRead; -use dep::types::mocked::{AggregationObject, Proof}; -use dep::types::constants::{ - NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, - NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, - CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, - PUBLIC_DATA_TREE_HEIGHT, - MAX_NEW_CONTRACTS_PER_TX, - NOTE_HASH_SUBTREE_HEIGHT, - CONTRACT_SUBTREE_HEIGHT, - NUM_FIELDS_PER_SHA256, - MAX_NEW_COMMITMENTS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - MAX_PUBLIC_DATA_READS_PER_TX, - MAX_NEW_NULLIFIERS_PER_TX, - NUM_ENCRYPTED_LOGS_HASHES_PER_TX, - MAX_NEW_L2_TO_L1_MSGS_PER_TX, - NUM_UNENCRYPTED_LOGS_HASHES_PER_TX, - NULLIFIER_SUBTREE_HEIGHT, - NULLIFIER_TREE_HEIGHT, - PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, - PUBLIC_DATA_SUBTREE_HEIGHT, +use crate::{ + abis::{ + constant_rollup_data::ConstantRollupData, + base_or_merge_rollup_public_inputs::{ + BaseOrMergeRollupPublicInputs, + BASE_ROLLUP_TYPE, + }, + }, + base::state_diff_hints::StateDiffHints, + components, + merkle_tree::{ + calculate_empty_tree_root, + calculate_subtree, + }, +}; +use dep::types::{ + abis::{ + append_only_tree_snapshot::AppendOnlyTreeSnapshot, + membership_witness::{ + ArchiveRootMembershipWitness, + MembershipWitness, + NullifierMembershipWitness, + PublicDataMembershipWitness, + }, + nullifier_leaf_preimage::NullifierLeafPreimage, + public_data_update_request::PublicDataUpdateRequest, + public_data_read::PublicDataRead, + previous_kernel_data::PreviousKernelData, + side_effect::{ + SideEffect, + SideEffectLinkedToNoteHash, + }, + }, + constants::{ + NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, + NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, + CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, + PUBLIC_DATA_TREE_HEIGHT, + MAX_NEW_CONTRACTS_PER_TX, + NOTE_HASH_SUBTREE_HEIGHT, + CONTRACT_SUBTREE_HEIGHT, + NUM_FIELDS_PER_SHA256, + MAX_NEW_COMMITMENTS_PER_TX, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + MAX_PUBLIC_DATA_READS_PER_TX, + MAX_NEW_NULLIFIERS_PER_TX, + NUM_ENCRYPTED_LOGS_HASHES_PER_TX, + MAX_NEW_L2_TO_L1_MSGS_PER_TX, + NUM_UNENCRYPTED_LOGS_HASHES_PER_TX, + NULLIFIER_SUBTREE_HEIGHT, + NULLIFIER_TREE_HEIGHT, + PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, + PUBLIC_DATA_SUBTREE_HEIGHT, + }, + mocked::{ + AggregationObject, + Proof + }, + partial_state_reference::PartialStateReference, + public_data_tree_leaf::PublicDataTreeLeaf, + public_data_tree_leaf_preimage::PublicDataTreeLeafPreimage, + utils::{ + field::{ + full_field_less_than, + full_field_greater_than, + }, + uint256::U256, + }, }; -use dep::types::abis::previous_kernel_data::PreviousKernelData; -use dep::types::abis::membership_witness::{NullifierMembershipWitness, PublicDataMembershipWitness, MembershipWitness}; -use dep::types::abis::membership_witness::ArchiveRootMembershipWitness; -use dep::types::abis::side_effect::{SideEffect, SideEffectLinkedToNoteHash}; -use dep::types::abis::nullifier_leaf_preimage::NullifierLeafPreimage; -use dep::types::partial_state_reference::PartialStateReference; struct BaseRollupInputs { kernel_data: PreviousKernelData, @@ -520,15 +551,6 @@ fn consistent_call_data_hash_full_fields() { ); } -// TODO to radix returns u8, so we cannot use bigger radixes. It'd be ideal to use a radix of the maximum range-constrained integer noir supports -pub fn full_field_less_than(lhs: Field, rhs: Field) -> bool { - lhs.lt(rhs) -} - -pub fn full_field_greater_than(lhs: Field, rhs: Field) -> bool { - rhs.lt(lhs) -} - #[test] fn test_u256_less_than() { assert(full_field_less_than(1, 1000)); @@ -549,58 +571,77 @@ fn test_u256_greater_than() { mod tests { use crate::{ + abis::{ + constant_rollup_data::ConstantRollupData, + base_or_merge_rollup_public_inputs::BaseOrMergeRollupPublicInputs, + }, base::{ + state_diff_hints::StateDiffHints, base_rollup_inputs::{ CALL_DATA_HASH_FULL_FIELDS, CALL_DATA_HASH_LOG_FIELDS, BaseRollupInputs, - full_field_less_than, }, - state_diff_hints::StateDiffHints, }, - merkle_tree::{calculate_subtree, calculate_empty_tree_root}, - abis::base_or_merge_rollup_public_inputs::BaseOrMergeRollupPublicInputs, - abis::public_data_tree_leaf::{PublicDataTreeLeafPreimage, PublicDataTreeLeaf}, - abis::constant_rollup_data::ConstantRollupData, - tests::merkle_tree_utils::{NonEmptyMerkleTree, compute_zero_hashes}, + merkle_tree::{ + calculate_empty_tree_root, + calculate_subtree, + }, components, - }; - use dep::types::constants::{ - CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, - CONTRACT_TREE_HEIGHT, - CONTRACT_SUBTREE_HEIGHT, - ARCHIVE_HEIGHT, - MAX_PUBLIC_DATA_READS_PER_TX, - MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - MAX_NEW_COMMITMENTS_PER_TX, - MAX_NEW_NULLIFIERS_PER_TX, - MAX_NEW_CONTRACTS_PER_TX, - NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, - NOTE_HASH_TREE_HEIGHT, - NOTE_HASH_SUBTREE_HEIGHT, - NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, - NULLIFIER_TREE_HEIGHT, - NULLIFIER_SUBTREE_HEIGHT, - PUBLIC_DATA_TREE_HEIGHT, - PUBLIC_DATA_SUBTREE_HEIGHT, - PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, - NUM_FIELDS_PER_SHA256, + tests::merkle_tree_utils::{ + NonEmptyMerkleTree, + compute_zero_hashes, + }, }; use dep::types::{ - abis::append_only_tree_snapshot::AppendOnlyTreeSnapshot, - abis::membership_witness::ArchiveRootMembershipWitness, - abis::membership_witness::{NullifierMembershipWitness, PublicDataMembershipWitness}, - abis::new_contract_data::NewContractData, - abis::nullifier_leaf_preimage::NullifierLeafPreimage, - abis::public_data_read::PublicDataRead, - abis::public_data_update_request::PublicDataUpdateRequest, - abis::previous_kernel_data::PreviousKernelData, - abis::side_effect::SideEffect, - tests::previous_kernel_data_builder::PreviousKernelDataBuilder, - address::{AztecAddress, EthAddress}, + abis::{ + append_only_tree_snapshot::AppendOnlyTreeSnapshot, + membership_witness::{ + ArchiveRootMembershipWitness, + NullifierMembershipWitness, + PublicDataMembershipWitness, + }, + new_contract_data::NewContractData, + nullifier_leaf_preimage::NullifierLeafPreimage, + public_data_read::PublicDataRead, + public_data_update_request::PublicDataUpdateRequest, + previous_kernel_data::PreviousKernelData, + side_effect::SideEffect, + }, + address::{ + AztecAddress, + EthAddress, + }, + constants::{ + CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, + CONTRACT_TREE_HEIGHT, + CONTRACT_SUBTREE_HEIGHT, + ARCHIVE_HEIGHT, + MAX_PUBLIC_DATA_READS_PER_TX, + MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, + MAX_NEW_COMMITMENTS_PER_TX, + MAX_NEW_NULLIFIERS_PER_TX, + MAX_NEW_CONTRACTS_PER_TX, + NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH, + NOTE_HASH_TREE_HEIGHT, + NOTE_HASH_SUBTREE_HEIGHT, + NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, + NULLIFIER_TREE_HEIGHT, + NULLIFIER_SUBTREE_HEIGHT, + PUBLIC_DATA_TREE_HEIGHT, + PUBLIC_DATA_SUBTREE_HEIGHT, + PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH, + NUM_FIELDS_PER_SHA256, + }, contract_class::ContractClassId, - utils::uint256::U256, partial_state_reference::PartialStateReference, + public_data_tree_leaf::PublicDataTreeLeaf, + public_data_tree_leaf_preimage::PublicDataTreeLeafPreimage, + tests::previous_kernel_data_builder::PreviousKernelDataBuilder, + utils::{ + field::full_field_less_than, + uint256::U256, + } }; use dep::std::option::Option; diff --git a/yarn-project/noir-protocol-circuits/src/crates/types/src/lib.nr b/yarn-project/noir-protocol-circuits/src/crates/types/src/lib.nr index b60e165e4fa3..19d13efc03c6 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/types/src/lib.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/types/src/lib.nr @@ -24,5 +24,7 @@ mod tests; mod state_reference; mod partial_state_reference; +mod public_data_tree_leaf; +mod public_data_tree_leaf_preimage; use abis::kernel_circuit_public_inputs::{ KernelCircuitPublicInputs, KernelCircuitPublicInputsFinal }; diff --git a/yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf.nr b/yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf.nr new file mode 100644 index 000000000000..57eb1945a3ac --- /dev/null +++ b/yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf.nr @@ -0,0 +1,28 @@ +use crate::traits::Empty; +use dep::std::cmp::Eq; + +struct PublicDataTreeLeaf { + slot: Field, + value: Field, +} + +impl Eq for PublicDataTreeLeaf { + fn eq(self, other: Self) -> bool { + (self.slot == other.slot) & (self.value == other.value) + } +} + +impl Empty for PublicDataTreeLeaf { + fn empty() -> Self { + Self { + slot: 0, + value: 0, + } + } +} + +impl PublicDataTreeLeaf { + pub fn is_empty(self) -> bool { + (self.slot == 0) & (self.value == 0) + } +} diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/public_data_tree_leaf.nr b/yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf_preimage.nr similarity index 58% rename from yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/public_data_tree_leaf.nr rename to yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf_preimage.nr index 0bd347a61165..dc84f6ee903c 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/public_data_tree_leaf.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/types/src/public_data_tree_leaf_preimage.nr @@ -1,5 +1,4 @@ -use dep::std::cmp::Eq; -use dep::types::traits::{Empty, Hash}; +use crate::traits::{Empty, Hash}; struct PublicDataTreeLeafPreimage { slot : Field, @@ -34,29 +33,3 @@ impl PublicDataTreeLeafPreimage { (self.slot == 0) & (self.value == 0) & (self.next_slot == 0) & (self.next_index == 0) } } - -struct PublicDataTreeLeaf { - slot: Field, - value: Field, -} - -impl Eq for PublicDataTreeLeaf { - fn eq(self, other: Self) -> bool { - (self.slot == other.slot) & (self.value == other.value) - } -} - -impl Empty for PublicDataTreeLeaf { - fn empty() -> Self { - Self { - slot: 0, - value: 0, - } - } -} - -impl PublicDataTreeLeaf { - pub fn is_empty(self) -> bool { - (self.slot == 0) & (self.value == 0) - } -} diff --git a/yarn-project/noir-protocol-circuits/src/crates/types/src/utils.nr b/yarn-project/noir-protocol-circuits/src/crates/types/src/utils.nr index b39707c73ae7..7b1496afb460 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/types/src/utils.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/types/src/utils.nr @@ -12,7 +12,6 @@ pub fn conditional_assign(predicate: bool, lhs: Field, rhs: Field) -> Field { if predicate { lhs } else { rhs } } -// Copied over from "yarn-project/aztec-nr/aztec/src/utils.nr" pub fn arr_copy_slice(src: [T; N], mut dst: [T; M], offset: Field) -> [T; M] { for i in 0..dst.len() { dst[i] = src[i + offset]; diff --git a/yarn-project/noir-protocol-circuits/src/crates/types/src/utils/field.nr b/yarn-project/noir-protocol-circuits/src/crates/types/src/utils/field.nr index 918543b66657..d5033666f1e1 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/types/src/utils/field.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/types/src/utils/field.nr @@ -13,3 +13,12 @@ pub fn field_from_bytes(bytes: [u8; N], big_endian: bool) -> Field { as_field } + +// TODO to radix returns u8, so we cannot use bigger radixes. It'd be ideal to use a radix of the maximum range-constrained integer noir supports +pub fn full_field_less_than(lhs: Field, rhs: Field) -> bool { + lhs.lt(rhs) +} + +pub fn full_field_greater_than(lhs: Field, rhs: Field) -> bool { + rhs.lt(lhs) +} From 47d66f9119cad1a82b621d30acab88f47886e063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Fri, 9 Feb 2024 17:06:18 +0100 Subject: [PATCH 3/3] refactor: aligning some naming in `BaseOrMergeRollupPublicInputs` (#4510) Partially fixes #3849 **Note**: Not fully tackling the issue here because the yellow paper needs to be updated first. --- .../src/abis/base_or_merge_rollup_public_inputs.nr | 3 +-- .../rollup-lib/src/base/base_rollup_inputs.nr | 4 ++-- .../src/crates/rollup-lib/src/components.nr | 4 ++-- .../rollup-lib/src/merge/merge_rollup_inputs.nr | 14 +++++++------- .../rollup-lib/src/tests/previous_rollup_data.nr | 4 ++-- .../noir-protocol-circuits/src/type_conversion.ts | 4 ++-- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/base_or_merge_rollup_public_inputs.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/base_or_merge_rollup_public_inputs.nr index 3cb598849cfd..9fd46ecdf0de 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/base_or_merge_rollup_public_inputs.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/base_or_merge_rollup_public_inputs.nr @@ -15,8 +15,7 @@ struct BaseOrMergeRollupPublicInputs { rollup_type : u32, // subtree height is always 0 for base. // so that we always pass-in two base/merge circuits of the same height into the next level of recursion - // TODO(benesjan): rename to height_in_block_tree - rollup_subtree_height : Field, + height_in_block_tree : Field, aggregation_object : AggregationObject, constants : ConstantRollupData, diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr index 73099641c563..3d8a6ab17a85 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr @@ -141,7 +141,7 @@ impl BaseRollupInputs { BaseOrMergeRollupPublicInputs { rollup_type : BASE_ROLLUP_TYPE, - rollup_subtree_height : 0, + height_in_block_tree : 0, aggregation_object : aggregation_object, constants : self.constants, start: self.start, @@ -1300,7 +1300,7 @@ mod tests { unconstrained fn subtree_height_is_0() { let outputs = BaseRollupInputsBuilder::new().execute(); - assert_eq(outputs.rollup_subtree_height, 0); + assert_eq(outputs.height_in_block_tree, 0); } #[test] diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/components.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/components.nr index fd8f28e6d068..a634fc74e70c 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/components.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/components.nr @@ -40,9 +40,9 @@ pub fn assert_both_input_proofs_of_same_height_and_return( right: BaseOrMergeRollupPublicInputs ) -> Field { assert( - left.rollup_subtree_height == right.rollup_subtree_height, "input proofs are of different rollup heights" + left.height_in_block_tree == right.height_in_block_tree, "input proofs are of different rollup heights" ); - left.rollup_subtree_height + left.height_in_block_tree } /** diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/merge/merge_rollup_inputs.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/merge/merge_rollup_inputs.nr index 9188ff8abb64..e3766db20cd7 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/merge/merge_rollup_inputs.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/merge/merge_rollup_inputs.nr @@ -30,7 +30,7 @@ impl MergeRollupInputs { let public_inputs = BaseOrMergeRollupPublicInputs { rollup_type : MERGE_ROLLUP_TYPE, - rollup_subtree_height : current_height + 1, + height_in_block_tree : current_height + 1, aggregation_object : aggregation_object, constants : left.constants, start : left.start, @@ -60,8 +60,8 @@ mod tests { #[test(should_fail_with="input proofs are of different rollup heights")] fn different_height_fails() { let mut inputs = default_merge_rollup_inputs(); - inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_subtree_height = 0; - inputs.previous_rollup_data[1].base_or_merge_rollup_public_inputs.rollup_subtree_height = 1; + inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.height_in_block_tree = 0; + inputs.previous_rollup_data[1].base_or_merge_rollup_public_inputs.height_in_block_tree = 1; let _output = inputs.merge_rollup_circuit(); } @@ -111,19 +111,19 @@ mod tests { let mut outputs = inputs.merge_rollup_circuit(); assert_eq(outputs.rollup_type, 1); assert_eq( - outputs.rollup_subtree_height, inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_subtree_height + 1 + outputs.height_in_block_tree, inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.height_in_block_tree + 1 ); // set inputs to have a merge rollup type and set the rollup height and test again. inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_type = 1; - inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_subtree_height = 1; + inputs.previous_rollup_data[0].base_or_merge_rollup_public_inputs.height_in_block_tree = 1; inputs.previous_rollup_data[1].base_or_merge_rollup_public_inputs.rollup_type = 1; - inputs.previous_rollup_data[1].base_or_merge_rollup_public_inputs.rollup_subtree_height = 1; + inputs.previous_rollup_data[1].base_or_merge_rollup_public_inputs.height_in_block_tree = 1; outputs = inputs.merge_rollup_circuit(); assert_eq(outputs.rollup_type, 1); - assert_eq(outputs.rollup_subtree_height, 2); + assert_eq(outputs.height_in_block_tree, 2); } #[test] diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/tests/previous_rollup_data.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/tests/previous_rollup_data.nr index e343d446d48e..99fc727568b3 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/tests/previous_rollup_data.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/tests/previous_rollup_data.nr @@ -76,8 +76,8 @@ pub fn default_previous_rollup_data() -> [PreviousRollupData; 2] { previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_type = BASE_ROLLUP_TYPE; previous_rollup_data[1].base_or_merge_rollup_public_inputs.rollup_type = BASE_ROLLUP_TYPE; - previous_rollup_data[0].base_or_merge_rollup_public_inputs.rollup_subtree_height = 1; - previous_rollup_data[1].base_or_merge_rollup_public_inputs.rollup_subtree_height = 1; + previous_rollup_data[0].base_or_merge_rollup_public_inputs.height_in_block_tree = 1; + previous_rollup_data[1].base_or_merge_rollup_public_inputs.height_in_block_tree = 1; previous_rollup_data[0].base_or_merge_rollup_public_inputs.calldata_hash = [0, 1]; previous_rollup_data[1].base_or_merge_rollup_public_inputs.calldata_hash = [2, 3]; diff --git a/yarn-project/noir-protocol-circuits/src/type_conversion.ts b/yarn-project/noir-protocol-circuits/src/type_conversion.ts index f7a2b1a845a7..8d0543cd37df 100644 --- a/yarn-project/noir-protocol-circuits/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits/src/type_conversion.ts @@ -1223,7 +1223,7 @@ export function mapBaseOrMergeRollupPublicInputsToNoir( ): BaseOrMergeRollupPublicInputsNoir { return { rollup_type: mapFieldToNoir(new Fr(baseOrMergeRollupPublicInputs.rollupType)), - rollup_subtree_height: mapFieldToNoir(new Fr(baseOrMergeRollupPublicInputs.rollupSubtreeHeight)), + height_in_block_tree: mapFieldToNoir(new Fr(baseOrMergeRollupPublicInputs.rollupSubtreeHeight)), aggregation_object: {}, constants: mapConstantRollupDataToNoir(baseOrMergeRollupPublicInputs.constants), start: mapPartialStateReferenceToNoir(baseOrMergeRollupPublicInputs.start), @@ -1270,7 +1270,7 @@ export function mapBaseOrMergeRollupPublicInputsFromNoir( ): BaseOrMergeRollupPublicInputs { return new BaseOrMergeRollupPublicInputs( mapNumberFromNoir(baseOrMergeRollupPublicInputs.rollup_type), - mapFieldFromNoir(baseOrMergeRollupPublicInputs.rollup_subtree_height), + mapFieldFromNoir(baseOrMergeRollupPublicInputs.height_in_block_tree), AggregationObject.makeFake(), mapConstantRollupDataFromNoir(baseOrMergeRollupPublicInputs.constants), mapPartialStateReferenceFromNoir(baseOrMergeRollupPublicInputs.start),