Don't store output_id in locked_outpust table, table stays ordered

src/blockchain_db/blockchain_db.cpp

@@ -41,6 +41,55 @@
 
 using epee::string_tools::pod_to_hex;
 
+//---------------------------------------------------------------
+// Helper function to group outputs by unlock block
+static void get_outs_by_unlock_block(const cryptonote::transaction &tx,
+  const std::vector<uint64_t> &output_ids,
+  const uint64_t tx_height,
+  const bool miner_tx,
+  fcmp_pp::curve_trees::OutputsByUnlockBlock &outs_by_unlock_block_inout)
+{
+  const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, tx_height);
+
+  CHECK_AND_ASSERT_THROW_MES(tx.vout.size() == output_ids.size(), "unexpected size of output ids");
+
+  for (std::size_t i = 0; i < tx.vout.size(); ++i)
+  {
+    const auto &out = tx.vout[i];
+
+    crypto::public_key output_public_key;
+    if (!cryptonote::get_output_public_key(out, output_public_key))
+      throw std::runtime_error("Could not get an output public key from a tx output.");
+
+    static_assert(CURRENT_TRANSACTION_VERSION == 2, "This section of code was written with 2 tx versions in mind. "
+      "Revisit this section and update for the new tx version.");
+    CHECK_AND_ASSERT_THROW_MES(tx.version == 1 || tx.version == 2, "encountered unexpected tx version");
+
+    if (!miner_tx && tx.version == 2)
+      CHECK_AND_ASSERT_THROW_MES(tx.rct_signatures.outPk.size() > i, "unexpected size of outPk");
+
+    rct::key commitment = (miner_tx || tx.version != 2)
+      ? rct::zeroCommit(out.amount) // Needs ringct
+      : tx.rct_signatures.outPk[i].mask;
+
+    auto output_pair = fcmp_pp::curve_trees::OutputPair{
+        .output_pubkey = std::move(output_public_key),
+        .commitment =    std::move(commitment)
+      };
+
+    if (outs_by_unlock_block_inout.find(unlock_block) == outs_by_unlock_block_inout.end())
+    {
+      auto new_vec = std::vector<fcmp_pp::curve_trees::OutputPair>{std::move(output_pair)};
+      outs_by_unlock_block_inout[unlock_block] = std::move(new_vec);
+    }
+    else
+    {
+      outs_by_unlock_block_inout[unlock_block].emplace_back(std::move(output_pair));
+    }
+  }
+}
+//---------------------------------------------------------------
+
 namespace cryptonote
 {
 
@@ -231,7 +280,7 @@ std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash
   {
     // miner v2 txes have their coinbase output in one single out to save space,
     // and we store them as rct outputs with an identity mask
-    // note: tx_outs_to_leaf_tuple_contexts in curve_trees.cpp mirrors this logic
+    // note: get_outs_by_unlock_block mirrors this logic
     if (miner_tx && tx.version == 2)
     {
       cryptonote::tx_out vout = tx.vout[i];
@@ -311,35 +360,32 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
   TIME_MEASURE_FINISH(time1);
   time_add_transaction += time1;
 
-  // When adding a block, we also need to add all the leaf tuples included in
-  // the block to a table keeping track of locked leaf tuples. Once those leaf
-  // tuples unlock, we use them to grow the tree.
-  std::multimap<uint64_t, fcmp_pp::curve_trees::LeafTupleContext> leaf_tuples_by_unlock_block;
+  // When adding a block, we also need to keep track of when outputs unlock, so
+  // we can use them to grow the merkle tree used in fcmp's at that point.
+  fcmp_pp::curve_trees::OutputsByUnlockBlock outs_by_unlock_block;
 
   // Get miner tx's leaf tuples
-  CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set");
-  m_curve_trees->tx_outs_to_leaf_tuple_contexts(
+  get_outs_by_unlock_block(
     blk.miner_tx,
     miner_output_ids,
     prev_height,
     true/*miner_tx*/,
-    leaf_tuples_by_unlock_block);
+    outs_by_unlock_block);
 
   // Get all other txs' leaf tuples
   for (std::size_t i = 0; i < txs.size(); ++i)
   {
-    // TODO: this loop can be parallelized
-    m_curve_trees->tx_outs_to_leaf_tuple_contexts(
+    get_outs_by_unlock_block(
       txs[i].first,
       output_ids[i],
       prev_height,
       false/*miner_tx*/,
-      leaf_tuples_by_unlock_block);
+      outs_by_unlock_block);
   }
 
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash, leaf_tuples_by_unlock_block);
+  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash, outs_by_unlock_block);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 

src/blockchain_db/blockchain_db.h

@@ -409,7 +409,7 @@ class BlockchainDB
    * @param cumulative_difficulty the accumulated difficulty after this block
    * @param coins_generated the number of coins generated total after this block
    * @param blk_hash the hash of the block
-   * @param leaf_tuples_by_unlock_block the leaves from this block to add to the merkle tree
+   * @param outs_by_unlock_block the outputs from this block to add to the merkle tree
    */
   virtual void add_block( const block& blk
                 , size_t block_weight
@@ -418,7 +418,7 @@ class BlockchainDB
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
                 , const crypto::hash& blk_hash
-                , const std::multimap<uint64_t, fcmp_pp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
+                , const fcmp_pp::curve_trees::OutputsByUnlockBlock& outs_by_unlock_block
                 ) = 0;
 
   /**
@@ -1783,7 +1783,7 @@ class BlockchainDB
   virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
 
   // TODO: description and make private
-  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::LeafTupleContext> &&new_leaves) = 0;
+  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::OutputPair> &&new_leaves) = 0;
 
   virtual void trim_tree(const uint64_t trim_n_leaf_tuples) = 0;
 

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -199,7 +199,7 @@ namespace
  *
  * spent_keys       input hash   -
  *
- * locked_outputs   block ID     [{output ID, output pubkey, commitment}...]
+ * locked_outputs   block ID     [{output pubkey, commitment}...]
  * leaves           leaf_idx     {output pubkey, commitment}
  * layers           layer_idx    [{child_chunk_idx, child_chunk_hash}...]
  *
@@ -798,7 +798,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin
 }
 
 void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
-    uint64_t num_rct_outs, const crypto::hash& blk_hash, const std::multimap<uint64_t, fcmp_pp::curve_trees::LeafTupleContext> &leaf_tuples_by_unlock_block)
+    uint64_t num_rct_outs, const crypto::hash& blk_hash, const fcmp_pp::curve_trees::OutputsByUnlockBlock& outs_by_unlock_block)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -828,7 +828,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
 
   // Grow the tree with outputs that unlock at this block height
   auto unlocked_outputs = this->get_outs_at_unlock_block_id(m_height);
-  this->grow_tree(std::move(unlocked_outputs));
+  if (!unlocked_outputs.empty())
+    this->grow_tree(std::move(unlocked_outputs));
 
   // Now that we've used the unlocked leaves to grow the tree, we can delete them from the locked outputs table
   this->del_locked_outs_at_block_id(m_height);
@@ -881,14 +882,13 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
   CURSOR(locked_outputs)
 
   // Add the locked outputs from this block to the locked outputs table
-  for (const auto &locked_output : leaf_tuples_by_unlock_block)
+  for (const auto &locked_output : outs_by_unlock_block)
   {
     MDB_val_set(k_block_id, locked_output.first);
     MDB_val_set(v_output, locked_output.second);
 
     // MDB_NODUPDATA because no benefit to having duplicate outputs in the tree, only 1 can be spent
-    // Can't use MDB_APPENDDUP because outputs aren't inserted in order sorted by unlock height
-    result = mdb_cursor_put(m_cur_locked_outputs, &k_block_id, &v_output, MDB_NODUPDATA);
+    result = mdb_cursor_put(m_cur_locked_outputs, &k_block_id, &v_output, MDB_APPENDDUP | MDB_NODUPDATA);
     if (result != MDB_SUCCESS && result != MDB_KEYEXIST)
       throw0(DB_ERROR(lmdb_error("Failed to add locked output: ", result).c_str()));
   }
@@ -1344,7 +1344,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
   }
 }
 
-void BlockchainLMDB::grow_tree(std::vector<fcmp_pp::curve_trees::LeafTupleContext> &&new_leaves)
+void BlockchainLMDB::grow_tree(std::vector<fcmp_pp::curve_trees::OutputPair> &&new_leaves)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   if (new_leaves.empty())
@@ -2225,7 +2225,7 @@ bool BlockchainLMDB::audit_layer(const std::unique_ptr<C_CHILD> &c_child,
   return audit_complete;
 }
 
-std::vector<fcmp_pp::curve_trees::LeafTupleContext> BlockchainLMDB::get_outs_at_unlock_block_id(
+std::vector<fcmp_pp::curve_trees::OutputPair> BlockchainLMDB::get_outs_at_unlock_block_id(
   uint64_t block_id)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -2238,7 +2238,7 @@ std::vector<fcmp_pp::curve_trees::LeafTupleContext> BlockchainLMDB::get_outs_at_
   MDB_val v_output;
 
   // Get all the locked outputs at the provided block id
-  std::vector<fcmp_pp::curve_trees::LeafTupleContext> leaf_tuples;
+  std::vector<fcmp_pp::curve_trees::OutputPair> outs;
 
   MDB_cursor_op op = MDB_SET;
   while (1)
@@ -2254,25 +2254,25 @@ std::vector<fcmp_pp::curve_trees::LeafTupleContext> BlockchainLMDB::get_outs_at_
     if (blk_id != block_id)
       throw0(DB_ERROR(("Blk id " + std::to_string(blk_id) + " not the expected" + std::to_string(block_id)).c_str()));
 
-    const auto range_begin = ((const fcmp_pp::curve_trees::LeafTupleContext*)v_output.mv_data);
-    const auto range_end = range_begin + v_output.mv_size / sizeof(fcmp_pp::curve_trees::LeafTupleContext);
+    const auto range_begin = ((const fcmp_pp::curve_trees::OutputPair*)v_output.mv_data);
+    const auto range_end = range_begin + v_output.mv_size / sizeof(fcmp_pp::curve_trees::OutputPair);
 
     auto it = range_begin;
 
     // The first MDB_NEXT_MULTIPLE includes the val from MDB_SET, so skip it
-    if (leaf_tuples.size() == 1)
+    if (outs.size() == 1)
       ++it;
 
     while (it < range_end)
     {
-      leaf_tuples.push_back(*it);
+      outs.push_back(*it);
       ++it;
     }
   }
 
   TXN_POSTFIX_RDONLY();
 
-  return leaf_tuples;
+  return outs;
 }
 
 void BlockchainLMDB::del_locked_outs_at_block_id(uint64_t block_id)
@@ -6867,24 +6867,25 @@ void BlockchainLMDB::migrate_5_6()
         }
 
         // Prepare the output for insertion to the tree
-        const auto tuple_context = m_curve_trees->output_to_leaf_context(output_id,
-          std::move(output_data.pubkey),
-          std::move(output_data.commitment));
+        const auto output_pair = fcmp_pp::curve_trees::OutputPair{
+            .output_pubkey = std::move(output_data.pubkey),
+            .commitment =    std::move(output_data.commitment)
+          };
 
         // Get the block in which the output will unlock
         const uint64_t unlock_block = cryptonote::get_unlock_block_index(output_data.unlock_time, output_data.height);
 
         // Now add the output to the locked outputs table
         MDB_val_set(k_block_id, unlock_block);
-        MDB_val_set(v_output, tuple_context);
+        MDB_val_set(v_output, output_pair);
 
         // MDB_NODUPDATA because no benefit to having duplicate outputs in the tree, only 1 can be spent
         // Can't use MDB_APPENDDUP because outputs aren't inserted in order sorted by unlock height
         result = mdb_cursor_put(c_locked_outputs, &k_block_id, &v_output, MDB_NODUPDATA);
         if (result != MDB_SUCCESS && result != MDB_KEYEXIST)
           throw0(DB_ERROR(lmdb_error("Failed to add locked output: ", result).c_str()));
         if (result == MDB_KEYEXIST)
-          MDEBUG("Dup output pubkey: " << tuple_context.output_pair.output_pubkey << " , output_id: " << output_id);
+          MDEBUG("Duplicate output pubkey: " << output_pair.output_pubkey << " , output_id: " << output_id);
       }
     }
 

src/blockchain_db/lmdb/db_lmdb.h

@@ -368,7 +368,7 @@ class BlockchainLMDB : public BlockchainDB
   static int compare_string(const MDB_val *a, const MDB_val *b);
 
   // make private
-  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::LeafTupleContext> &&new_leaves);
+  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::OutputPair> &&new_leaves);
 
   virtual void trim_tree(const uint64_t trim_n_leaf_tuples);
 
@@ -388,7 +388,7 @@ class BlockchainLMDB : public BlockchainDB
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
                 , const crypto::hash& block_hash
-                , const std::multimap<uint64_t, fcmp_pp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
+                , const fcmp_pp::curve_trees::OutputsByUnlockBlock& outs_by_unlock_block
                 );
 
   virtual void remove_block();
@@ -447,7 +447,7 @@ class BlockchainLMDB : public BlockchainDB
     const uint64_t child_layer_idx,
     const uint64_t chunk_width) const;
 
-  std::vector<fcmp_pp::curve_trees::LeafTupleContext> get_outs_at_unlock_block_id(uint64_t block_id);
+  std::vector<fcmp_pp::curve_trees::OutputPair> get_outs_at_unlock_block_id(uint64_t block_id);
 
   void del_locked_outs_at_block_id(uint64_t block_id);
 

src/blockchain_db/testdb.h

@@ -116,7 +116,7 @@ class BaseTestDB: public cryptonote::BlockchainDB {
   virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
   virtual void add_spent_key(const crypto::key_image& k_image) override {}
   virtual void remove_spent_key(const crypto::key_image& k_image) override {}
-  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::LeafTupleContext> &&new_leaves) override {};
+  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::OutputPair> &&new_leaves) override {};
   virtual void trim_tree(const uint64_t trim_n_leaf_tuples) override {};
   virtual bool audit_tree(const uint64_t expected_n_leaf_tuples) const override { return false; };
   virtual std::array<uint8_t, 32UL> get_tree_root() const override { return {}; };
@@ -149,7 +149,7 @@ class BaseTestDB: public cryptonote::BlockchainDB {
                         , const uint64_t& coins_generated
                         , uint64_t num_rct_outs
                         , const crypto::hash& blk_hash
-                        , const std::multimap<uint64_t, fcmp_pp::curve_trees::LeafTupleContext>& leaf_tuples_by_unlock_block
+                        , const fcmp_pp::curve_trees::OutputsByUnlockBlock& outs_by_unlock_block
                         ) override { }
   virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); }
   virtual void set_hard_fork_version(uint64_t height, uint8_t version) override {}

src/cryptonote_basic/cryptonote_format_utils.h

@@ -37,6 +37,7 @@
 #include "include_base_utils.h"
 #include "crypto/crypto.h"
 #include "crypto/hash.h"
+#include "fcmp_pp/curve_trees.h"
 #include <unordered_map>
 #include <boost/multiprecision/cpp_int.hpp>
 

src/fcmp_pp/CMakeLists.txt

@@ -51,7 +51,6 @@ endif()
 target_link_libraries(fcmp_pp
   PUBLIC
     cncrypto
-    cryptonote_basic
     epee
   PRIVATE
     ${CMAKE_CURRENT_BINARY_DIR}/fcmp_pp_rust/libfcmp_pp_rust.a