fcmp++ TreeSync: (de)ser TreeSyncMemory & start wallet2 integration

- Implemented serialization on TreeSyncMemory class
- Introduced TreeSyncMemory in wallet2

src/fcmp_pp/CMakeLists.txt

@@ -55,6 +55,7 @@ target_link_libraries(fcmp_pp
     common
     epee
   PRIVATE
+    ${Boost_SERIALIZATION_LIBRARY}
     ${CMAKE_CURRENT_BINARY_DIR}/fcmp_pp_rust/libfcmp_pp_rust.a
     ${EXTRA_LIBRARIES}
     ${EXTRA_RUST_LIBRARIES})

src/fcmp_pp/curve_trees.h

@@ -30,6 +30,7 @@
 
 #include "crypto/crypto.h"
 #include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/cryptonote_boost_serialization.h"
 #include "fcmp_pp_crypto.h"
 #include "misc_log_ex.h"
 #include "tower_cycle.h"
@@ -136,6 +137,18 @@ struct OutputPair final
 {
     crypto::public_key output_pubkey;
     rct::key           commitment;
+
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & output_pubkey;
+        a & commitment;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        FIELD(output_pubkey)
+        FIELD(commitment)
+    END_SERIALIZE()
 };
 
 // Contextual wrapper for the output

src/fcmp_pp/tree_sync.h

@@ -52,8 +52,8 @@ template<typename C1, typename C2>
 class TreeSync
 {
 public:
-    TreeSync(std::shared_ptr<CurveTrees<C1, C2>> &curve_trees,
-        const std::size_t max_reorg_depth = ORPHANED_BLOCKS_MAX_COUNT):
+    TreeSync(std::shared_ptr<CurveTrees<C1, C2>> curve_trees,
+        const uint64_t max_reorg_depth = ORPHANED_BLOCKS_MAX_COUNT):
             m_curve_trees{curve_trees},
             m_max_reorg_depth{max_reorg_depth}
     {};
@@ -85,10 +85,13 @@ class TreeSync
     // - Returns true with empty path_out if the output is registered but not yet included in the tree
     virtual bool get_output_path(const OutputPair &output, typename CurveTrees<C1, C2>::Path &path_out) const = 0;
 
+    // Overwrite the max reorg depth
+    void set_max_reorg_depth(const uint64_t max_reorg_depth) { m_max_reorg_depth = max_reorg_depth; };
+
 // Internal member variables accessible by derived class
 protected:
     std::shared_ptr<CurveTrees<C1, C2>> m_curve_trees;
-    const std::size_t m_max_reorg_depth;
+    uint64_t m_max_reorg_depth;
 };
 //----------------------------------------------------------------------------------------------------------------------
 //----------------------------------------------------------------------------------------------------------------------

src/fcmp_pp/tree_sync_memory.cpp

@@ -758,15 +758,15 @@ void TreeSyncMemory<C1, C2>::sync_block(const uint64_t block_idx,
     m_cached_blocks.push_back(std::move(blk_meta));
 
     // Deque the oldest cached block upon reaching the max reorg depth
-    if (m_cached_blocks.size() > TreeSync<C1, C2>::m_max_reorg_depth)
+    if ((uint64_t)m_cached_blocks.size() > TreeSync<C1, C2>::m_max_reorg_depth)
     {
         CHECK_AND_ASSERT_THROW_MES(!m_cached_blocks.empty(), "empty cached blocks");
         this->deque_block(m_cached_blocks.front());
         m_cached_blocks.pop_front();
     }
 
-    CHECK_AND_ASSERT_THROW_MES((TreeSync<C1, C2>::m_max_reorg_depth >= m_cached_blocks.size()),
-        "cached blocks exceeded max reorg depth");
+    if ((uint64_t)m_cached_blocks.size() >= TreeSync<C1, C2>::m_max_reorg_depth)
+        LOG_ERROR("Cached blocks exceeded max reorg depth");
 }
 
 // Explicit instantiation

src/fcmp_pp/tree_sync_memory.h

@@ -28,12 +28,22 @@
 
 #pragma once
 
+#include "common/unordered_containers_boost_serialization.h"
 #include "cryptonote_config.h"
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "curve_trees.h"
 #include "ringct/rctTypes.h"
+#include "serialization/array.h"
+#include "serialization/containers.h"
+#include "serialization/crypto.h"
+#include "serialization/pair.h"
+#include "serialization/serialization.h"
 #include "tree_sync.h"
 
+#include <boost/serialization/array.hpp>
+#include <boost/serialization/deque.hpp>
+#include <boost/serialization/vector.hpp>
+#include <boost/serialization/version.hpp>
 #include <deque>
 #include <memory>
 #include <unordered_map>
@@ -59,22 +69,60 @@ struct BlockMeta final
     BlockIdx blk_idx;
     BlockHash blk_hash;
     uint64_t n_leaf_tuples;
+
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & blk_hash;
+        a & blk_idx;
+        a & n_leaf_tuples;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        FIELD(blk_idx)
+        FIELD(blk_hash)
+        FIELD(n_leaf_tuples)
+    END_SERIALIZE()
 };
 
 // We need to use a ref count on all individual elems in the cache because it's possible for:
 //  a) multiple blocks to share path elems that need to remain after pruning a block past the max reorg depth.
 //  b) multiple registered outputs to share the same path elems.
 // We can't remove a cached elem unless we know it's ref'd 0 times.
-struct CachedTreeElemChunk final
+struct CachedLeafChunk final
 {
-    std::vector<std::array<uint8_t, 32UL>> tree_elems;
+    std::vector<OutputPair> leaves;
     uint64_t ref_count;
+
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & leaves;
+        a & ref_count;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        FIELD(leaves)
+        FIELD(ref_count)
+    END_SERIALIZE()
 };
 
-struct CachedLeafChunk final
+struct CachedTreeElemChunk final
 {
-    std::vector<OutputPair> leaves;
+    std::vector<std::array<uint8_t, 32UL>> tree_elems;
     uint64_t ref_count;
+
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & tree_elems;
+        a & ref_count;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        FIELD(tree_elems)
+        FIELD(ref_count)
+    END_SERIALIZE()
 };
 
 struct AssignedLeafIdx final
@@ -84,6 +132,18 @@ struct AssignedLeafIdx final
 
     void assign_leaf(const LeafIdx idx) { leaf_idx = idx; assigned_leaf_idx = true; }
     void unassign_leaf() { leaf_idx = 0; assigned_leaf_idx = false; }
+
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & assigned_leaf_idx;
+        a & leaf_idx;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        FIELD(assigned_leaf_idx)
+        FIELD(leaf_idx)
+    END_SERIALIZE()
 };
 
 using RegisteredOutputs = std::unordered_map<OutputRef, AssignedLeafIdx>;
@@ -106,8 +166,8 @@ template<typename C1, typename C2>
 class TreeSyncMemory final : public TreeSync<C1, C2>
 {
 public:
-    TreeSyncMemory(std::shared_ptr<CurveTrees<C1, C2>> &curve_trees,
-        const std::size_t max_reorg_depth = ORPHANED_BLOCKS_MAX_COUNT):
+    TreeSyncMemory(std::shared_ptr<CurveTrees<C1, C2>> curve_trees,
+        const uint64_t max_reorg_depth = ORPHANED_BLOCKS_MAX_COUNT):
             TreeSync<C1, C2>(curve_trees, max_reorg_depth)
     {};
 
@@ -150,9 +210,51 @@ class TreeSyncMemory final : public TreeSync<C1, C2>
     //   the tree extensions and reductions for each block correctly locally when syncing.
     std::deque<BlockMeta> m_cached_blocks;
 
-// TODO: serialization
+// Serialization
+public:
+    template <class Archive>
+    inline void serialize(Archive &a, const unsigned int ver)
+    {
+        a & m_registered_outputs;
+        a & m_leaf_cache;
+        a & m_tree_elem_cache;
+        a & m_cached_blocks;
+    }
+
+    BEGIN_SERIALIZE_OBJECT()
+        VERSION_FIELD(0)
+        FIELD(m_registered_outputs)
+        FIELD(m_leaf_cache)
+        FIELD(m_tree_elem_cache)
+        FIELD(m_cached_blocks)
+        // It's possible for m_cached_blocks.size() > m_max_reorg_depth if the max reorg depth changes across runs.
+        // This is ok as implemented. m_cached_blocks.size() will stay constant while syncing in this case.
+    END_SERIALIZE()
 };
 //----------------------------------------------------------------------------------------------------------------------
 //----------------------------------------------------------------------------------------------------------------------
 }//namespace curve_trees
 }//namespace fcmp_pp
+
+
+// Since BOOST_CLASS_VERSION does not work for templated class, implement it
+namespace boost
+{
+namespace serialization
+{
+template<typename C1, typename C2>
+struct version<fcmp_pp::curve_trees::TreeSyncMemory<C1, C2>>
+{
+    static const int VERSION = 0;
+    typedef mpl::int_<VERSION> type;
+    typedef mpl::integral_c_tag tag;
+    BOOST_STATIC_CONSTANT(unsigned int, value = version::type::value);
+    BOOST_MPL_ASSERT((
+        boost::mpl::less<
+            boost::mpl::int_<VERSION>,
+            boost::mpl::int_<256>
+        >
+    ));
+};
+}
+}

src/serialization/array.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2024, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+#include <array>
+#include <type_traits>
+#include "serialization.h"
+
+template <template <bool> class Archive, typename T, std::size_t SIZE>
+inline bool do_serialize(Archive<false>& ar, std::array<T, SIZE>& _array)
+{
+  static_assert(std::is_arithmetic<T>::value, "only serialize array of arithmetic types");
+  if (ar.remaining_bytes() < (sizeof(T) * SIZE))
+  {
+    ar.set_fail();
+    return false;
+  }
+  ar.serialize_blob(_array.data(), SIZE);
+  return true;
+}
+
+template <template <bool> class Archive, typename T, std::size_t SIZE>
+inline bool do_serialize(Archive<true>& ar, std::array<T, SIZE>& _array)
+{
+  static_assert(std::is_arithmetic<T>::value, "only serialize array of arithmetic types");
+  ar.serialize_blob(_array.data(), SIZE);
+  return true;
+}

src/wallet/CMakeLists.txt

@@ -52,6 +52,7 @@ target_link_libraries(wallet
     multisig
     common
     cryptonote_core
+    fcmp_pp
     mnemonics
     device_trezor
     net

src/wallet/wallet2.cpp

@@ -1256,7 +1256,10 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std
   m_enable_multisig(false),
   m_pool_info_query_time(0),
   m_has_ever_refreshed_from_node(false),
-  m_allow_mismatched_daemon_version(false)
+  m_allow_mismatched_daemon_version(false),
+  m_tree_sync(fcmp_pp::curve_trees::TreeSyncMemory<Helios, Selene>(
+      fcmp_pp::curve_trees::curve_trees_v1(),
+      m_max_reorg_depth))
 {
 }
 
@@ -6459,6 +6462,9 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
     THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer");
   }
 
+  // We may have loaded a max reorg depth different than the default
+  m_tree_sync.set_max_reorg_depth(m_max_reorg_depth);
+
   wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_is_background_wallet, password);
 
   //keys loaded ok!

src/wallet/wallet2.h

@@ -57,6 +57,8 @@
 #include "common/util.h"
 #include "crypto/chacha.h"
 #include "crypto/hash.h"
+#include "fcmp_pp/curve_trees.h"
+#include "fcmp_pp/tree_sync_memory.h"
 #include "multisig/multisig_account.h"
 #include "ringct/rctTypes.h"
 #include "ringct/rctOps.h"
@@ -1369,11 +1371,14 @@ namespace tools
         return;
       }
       a & m_background_sync_data;
+      if(ver < 32)
+        return;
+      a & m_tree_sync;
     }
 
     BEGIN_SERIALIZE_OBJECT()
       MAGIC_FIELD("monero wallet cache")
-      VERSION_FIELD(2)
+      VERSION_FIELD(3)
       FIELD(m_blockchain)
       FIELD(m_transfers)
       FIELD(m_account_public_address)
@@ -1412,6 +1417,9 @@ namespace tools
         return true;
       }
       FIELD(m_background_sync_data)
+      if (version < 3)
+        return true;
+      FIELD(m_tree_sync)
     END_SERIALIZE()
 
     /*!
@@ -2056,9 +2064,13 @@ namespace tools
     bool m_background_syncing;
     bool m_processing_background_cache;
     background_sync_data_t m_background_sync_data;
+
+    using Helios = fcmp_pp::curve_trees::Helios;
+    using Selene = fcmp_pp::curve_trees::Selene;
+    fcmp_pp::curve_trees::TreeSyncMemory<Helios, Selene> m_tree_sync;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 31)
+BOOST_CLASS_VERSION(tools::wallet2, 32)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 12)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)

tests/fuzz/CMakeLists.txt

@@ -193,6 +193,7 @@ target_link_libraries(bulletproof_fuzz_tests
     ${Boost_CHRONO_LIBRARY}
     ${Boost_REGEX_LIBRARY}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    ${Boost_SERIALIZATION_LIBRARY}
     ${CMAKE_THREAD_LIBS_INIT}
     ${EXTRA_LIBRARIES}
     $ENV{LIB_FUZZING_ENGINE})