Merge remote-tracking branch 'origin/master' into cg/two-folding-veri…

…fiers-test

.github/workflows/devnet-deploys.yml

@@ -771,7 +771,6 @@ jobs:
           ./.github/scripts/wait_for_infra.sh pxe ${{ env.DEPLOY_TAG }} ${{ env.API_KEY }}
 
       - name: Deploy verifier (allow failure)
-        continue-on-error: true
         working-directory: ./yarn-project/aztec/terraform/pxe
         run: |
           set -eo pipefail

avm-transpiler/src/opcodes.rs

@@ -26,12 +26,14 @@ pub enum AvmOpcode {
     OR_16,
     XOR_8,
     XOR_16,
-    NOT,
+    NOT_8,
+    NOT_16,
     SHL_8,
     SHL_16,
     SHR_8,
     SHR_16,
-    CAST,
+    CAST_8,
+    CAST_16,
     // Execution environment
     ADDRESS,
     STORAGEADDRESS,
@@ -127,13 +129,15 @@ impl AvmOpcode {
             AvmOpcode::OR_16 => "OR_16",
             AvmOpcode::XOR_8 => "XOR_8",
             AvmOpcode::XOR_16 => "XOR_16",
-            AvmOpcode::NOT => "NOT",
+            AvmOpcode::NOT_8 => "NOT_8",
+            AvmOpcode::NOT_16 => "NOT_16",
             AvmOpcode::SHL_8 => "SHL_8",
             AvmOpcode::SHL_16 => "SHL_16",
             AvmOpcode::SHR_8 => "SHR_8",
             AvmOpcode::SHR_16 => "SHR_16",
             // Compute - Type Conversions
-            AvmOpcode::CAST => "CAST",
+            AvmOpcode::CAST_8 => "CAST_8",
+            AvmOpcode::CAST_16 => "CAST_16",
 
             // Execution Environment
             AvmOpcode::ADDRESS => "ADDRESS",

avm-transpiler/src/transpile.rs

@@ -823,6 +823,12 @@ fn generate_cast_instruction(
     destination_indirect: bool,
     dst_tag: AvmTypeTag,
 ) -> AvmInstruction {
+    let bits_needed = bits_needed_for(&source).max(bits_needed_for(&destination));
+    let avm_opcode = match bits_needed {
+        8 => AvmOpcode::CAST_8,
+        16 => AvmOpcode::CAST_16,
+        _ => panic!("CAST only supports 8 and 16 bit encodings, needed {}", bits_needed),
+    };
     let mut indirect_flags = ALL_DIRECT;
     if source_indirect {
         indirect_flags |= ZEROTH_OPERAND_INDIRECT;
@@ -831,10 +837,10 @@ fn generate_cast_instruction(
         indirect_flags |= FIRST_OPERAND_INDIRECT;
     }
     AvmInstruction {
-        opcode: AvmOpcode::CAST,
+        opcode: avm_opcode,
         indirect: Some(indirect_flags),
         tag: Some(dst_tag),
-        operands: vec![AvmOperand::U32 { value: source }, AvmOperand::U32 { value: destination }],
+        operands: vec![make_operand(bits_needed, &source), make_operand(bits_needed, &destination)],
     }
 }
 

barretenberg/cpp/CMakeLists.txt

@@ -38,6 +38,7 @@ option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)
 # Note: Must do 'sudo apt-get install libdw-dev' or equivalent
 option(CHECK_CIRCUIT_STACKTRACES "Enable (slow) stack traces for check circuit" OFF)
 option(ENABLE_TRACY "Enable low-medium overhead profiling for memory and performance with tracy" OFF)
+option(ENABLE_PIC "Builds with position independent code" OFF)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
     message(STATUS "Compiling for ARM.")

barretenberg/cpp/CMakePresets.json

@@ -21,6 +21,23 @@
         "TARGET_ARCH": "skylake"
       }
     },
+    {
+      "name": "default-pic",
+      "displayName": "Build with Clang with Position Independent Code",
+      "description": "Build with globally installed Clang with Position Independent Code",
+      "binaryDir": "build-pic",
+      "generator": "Ninja",
+      "environment": {
+        "CC": "clang",
+        "CXX": "clang++",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
+        "LDFLAGS": "-Wl,-undefined,dynamic_lookup"
+      },
+      "cacheVariables": {
+        "TARGET_ARCH": "skylake",
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "homebrew",
       "displayName": "Homebrew + Clang",
@@ -66,6 +83,16 @@
         "CXX": "clang++-16"
       }
     },
+    {
+      "name": "clang16-pic",
+      "displayName": "Release build with Position Independent Code",
+      "description": "Build with globally installed Clang-16 using Position Independent Code",
+      "inherits": "clang16",
+      "binaryDir": "build-pic",
+      "cacheVariables": {
+        "ENABLE_PIC": "ON"
+      }
+    },
     {
       "name": "clang16-dbg",
       "displayName": "Debugging build with Clang-16",
@@ -446,6 +473,16 @@
       "inherits": "default",
       "configurePreset": "tracy"
     },
+    {
+      "name": "clang16-pic",
+      "inherits": "default",
+      "configurePreset": "clang16-pic"
+    },
+    {
+      "name": "default-pic",
+      "inherits": "default",
+      "configurePreset": "default-pic"
+    },
     {
       "name": "tracy-gates",
       "inherits": "default",

barretenberg/cpp/Earthfile

@@ -29,6 +29,11 @@ preset-release:
     RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src}
     SAVE ARTIFACT build/bin
 
+preset-release-world-state:
+  FROM +source
+  RUN cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && cp ./build/lib/world_state_napi.node ./build/bin && rm -rf build/{deps,lib,src}
+  SAVE ARTIFACT build/bin
+
 preset-release-assert:
     FROM +source
     RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src}
@@ -264,3 +269,4 @@ build:
     BUILD +preset-wasm
     BUILD +preset-wasm-threads
     BUILD +preset-release
+    BUILD +preset-release-world-state

barretenberg/cpp/bootstrap.sh

@@ -46,17 +46,27 @@ else
   fi
 fi
 
+PIC_PRESET="$PRESET-pic"
+
 # Remove cmake cache files.
 rm -f {build,build-wasm,build-wasm-threads}/CMakeCache.txt
 
+(cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
+
 echo "#################################"
 echo "# Building with preset: $PRESET"
 echo "# When running cmake directly, remember to use: --build --preset $PRESET"
 echo "#################################"
 
 function build_native {
+  # Build bb with standard preset and world_state_napi with Position Independent code variant
   cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
+  cmake --preset $PIC_PRESET -DCMAKE_BUILD_TYPE=RelWithAssert
   cmake --build --preset $PRESET --target bb
+  cmake --build --preset $PIC_PRESET --target world_state_napi
+  # copy the world_state_napi build artifact over to the world state in yarn-project
+  mkdir -p ../../yarn-project/world-state/build/
+  cp ./build-pic/lib/world_state_napi.node ../../yarn-project/world-state/build/
 }
 
 function build_wasm {
@@ -106,7 +116,7 @@ fi
 
 if [ ! -d ./srs_db/grumpkin ]; then
   # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests
-  # If tests require more points, the parameter can be increased here. Note: IPA requires 
+  # If tests require more points, the parameter can be increased here. Note: IPA requires
   # dyadic_circuit_size + 1 points so in general this number will be a power of two plus 1
   cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8193
 fi

barretenberg/cpp/scripts/world_state_tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+TEST=${1:-*}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target world_state_tests
+./build/bin/world_state_tests --gtest_filter=WorldStateTest.${TEST}

barretenberg/cpp/src/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 # Enable the following warnings project wide.
 # If any compilation issues arise in the future, they should not be silenced here but rather in the
@@ -53,6 +54,13 @@ else()
     message(STATUS "Using optimized assembly for field arithmetic.")
 endif()
 
+if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  message("Building with Position Independent Code")
+  set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
+  set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -fPIC")
+  add_subdirectory(barretenberg/world_state_napi)
+endif()
+
 add_subdirectory(barretenberg/aztec_ivc)
 add_subdirectory(barretenberg/bb)
 add_subdirectory(barretenberg/circuit_checker)
@@ -87,6 +95,7 @@ add_subdirectory(barretenberg/translator_vm)
 add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm)
 add_subdirectory(barretenberg/wasi)
+add_subdirectory(barretenberg/world_state)
 
 if(SMT)
     add_subdirectory(barretenberg/smt_verification)

barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp

@@ -382,9 +382,9 @@ TEST_F(IPATest, ShpleminiIPAWithShift)
         std::string label = "Gemini:a_" + std::to_string(l);
         const auto& evaluation = gemini_opening_pairs[l + 1].evaluation;
         prover_transcript->send_to_verifier(label, evaluation);
-        opening_claims.emplace_back(gemini_witnesses[l], gemini_opening_pairs[l]);
+        opening_claims.push_back({ gemini_witnesses[l], gemini_opening_pairs[l] });
     }
-    opening_claims.emplace_back(gemini_witnesses[log_n], gemini_opening_pairs[log_n]);
+    opening_claims.push_back({ gemini_witnesses[log_n], gemini_opening_pairs[log_n] });
 
     const auto opening_claim = ShplonkProver::prove(this->ck(), opening_claims, prover_transcript);
     IPA::compute_opening_proof(this->ck(), opening_claim, prover_transcript);

barretenberg/cpp/src/barretenberg/commitment_schemes/kzg/kzg.test.cpp

@@ -250,9 +250,9 @@ TYPED_TEST(KZGTest, ShpleminiKzgWithShift)
         std::string label = "Gemini:a_" + std::to_string(l);
         const auto& evaluation = gemini_opening_pairs[l + 1].evaluation;
         prover_transcript->send_to_verifier(label, evaluation);
-        opening_claims.emplace_back(gemini_witnesses[l], gemini_opening_pairs[l]);
+        opening_claims.push_back({ gemini_witnesses[l], gemini_opening_pairs[l] });
     }
-    opening_claims.emplace_back(gemini_witnesses[log_n], gemini_opening_pairs[log_n]);
+    opening_claims.push_back({ gemini_witnesses[log_n], gemini_opening_pairs[log_n] });
 
     // Shplonk prover output:
     // - opening pair: (z_challenge, 0)

barretenberg/cpp/src/barretenberg/commitment_schemes_recursion/shplemini.test.cpp

@@ -125,9 +125,9 @@ TEST(ShpleminiRecursionTest, ProveAndVerifySingle)
         std::string label = "Gemini:a_" + std::to_string(l);
         const auto& evaluation = gemini_opening_pairs[l + 1].evaluation;
         prover_transcript->send_to_verifier(label, evaluation);
-        opening_claims.emplace_back(gemini_witnesses[l], gemini_opening_pairs[l]);
+        opening_claims.push_back({ gemini_witnesses[l], gemini_opening_pairs[l] });
     }
-    opening_claims.emplace_back(gemini_witnesses[log_circuit_size], gemini_opening_pairs[log_circuit_size]);
+    opening_claims.push_back({ gemini_witnesses[log_circuit_size], gemini_opening_pairs[log_circuit_size] });
 
     // Shplonk prover output:
     // - opening pair: (z_challenge, 0)

barretenberg/cpp/src/barretenberg/common/constexpr_utils.hpp

@@ -5,6 +5,21 @@
 #include <utility>
 
 namespace bb {
+namespace detail {
+
+/**
+ * @brief Create an index sequence from Min to Max (not included) with an increment of Inc
+ */
+template <size_t Min, size_t Max, size_t Inc> constexpr auto make_index_range()
+{
+    static_assert(Max >= Min);
+    static_assert(Inc >= 1);
+    return []<size_t... Is>(std::index_sequence<Is...>) {
+        return std::index_sequence<Min + (Is * Inc)...>{};
+    }(std::make_index_sequence<(Max - Min - 1) / Inc + 1>{});
+}
+
+} // namespace detail
 
 /**
  * @brief Implements a loop using a compile-time iterator. Requires c++20.
@@ -55,37 +70,11 @@ namespace bb {
  */
 template <size_t Start, size_t End, size_t Inc, class F> constexpr void constexpr_for(F&& f)
 {
-    // Call function `f<Start>()` iff Start < End
-    if constexpr (Start < End) {
-        // F must be a template lambda with a single **typed** template parameter that represents the iterator
-        // (e.g. [&]<size_t i>(){ ... } is good)
-        // (and [&]<typename i>(){ ... } won't compile!)
-
-        /**
-         * Explaining f.template operator()<Start>()
-         *
-         * The following line must explicitly tell the compiler that <Start> is a template parameter by using the
-         * `template` keyword.
-         * (if we wrote f<Start>(), the compiler could legitimately interpret `<` as a less than symbol)
-         *
-         * The fragment `f.template` tells the compiler that we're calling a *templated* member of `f`.
-         * The "member" being called is the function operator, `operator()`, which must be explicitly provided
-         * (for any function X, `X(args)` is an alias for `X.operator()(args)`)
-         * The compiler has no alias `X.template <tparam>(args)` for `X.template operator()<tparam>(args)` so we must
-         * write it explicitly here
-         *
-         * To summarize what the next line tells the compiler...
-         * 1. I want to call a member of `f` that expects one or more template parameters
-         * 2. The member of `f` that I want to call is the function operator
-         * 3. The template parameter is `Start`
-         * 4. The function operator itself contains no arguments
-         */
-        f.template operator()<Start>();
-
-        // Once we have executed `f`, we recursively call the `constexpr_for` function, increasing the value of `Start`
-        // by `Inc`
-        constexpr_for<Start + Inc, End, Inc>(f);
-    }
+    // F must be a template lambda with a single **typed** template parameter that represents the iterator
+    // (e.g. [&]<size_t i>(){ ... } is good)
+    // (and [&]<typename i>(){ ... } won't compile!)
+    constexpr auto indices = detail::make_index_range<Start, End, Inc>();
+    [&]<size_t... Is>(std::index_sequence<Is...>) { (f.template operator()<Is>(), ...); }(indices);
 }
 
 }; // namespace bb

barretenberg/cpp/src/barretenberg/messaging/dispatcher.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "barretenberg/messaging/header.hpp"
+#include "barretenberg/serialize/cbind.hpp"
+#include <cstdint>
+#include <functional>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+namespace bb::messaging {
+
+using message_handler = std::function<bool(msgpack::object&, msgpack::sbuffer&)>;
+
+class MessageDispatcher {
+  private:
+    std::unordered_map<uint32_t, message_handler> messageHandlers;
+
+  public:
+    MessageDispatcher() = default;
+
+    bool onNewData(msgpack::object& obj, msgpack::sbuffer& buffer)
+    {
+        bb::messaging::HeaderOnlyMessage header;
+        obj.convert(header);
+
+        auto iter = messageHandlers.find(header.msgType);
+        if (iter == messageHandlers.end()) {
+            throw std::runtime_error("No registered handler for message of type " + std::to_string(header.msgType));
+        }
+
+        return (iter->second)(obj, buffer);
+    }
+
+    void registerTarget(uint32_t msgType, const message_handler& handler)
+    {
+        messageHandlers.insert({ msgType, handler });
+    }
+};
+
+} // namespace bb::messaging