fcmp++ TreeSync: sync chunks of blocks at a time correctly

src/blockchain_db/blockchain_db_utils.h

@@ -40,7 +40,7 @@ namespace cryptonote
 // These functions internally rely on ringct for zeroCommitVartime. I implemented in this blockchain_db_utils file
 // instead of cryptonote_basic (where it would seem the better place to put it) to avoid a circular dependency between
 // ringct <> cryptonote_basic.
-// Note that zeroCommitVartime causes this function to execute slowly.
+// Note that zeroCommitVartime is expensive.
 std::pair<fcmp_pp::curve_trees::OutputsByUnlockBlock, uint64_t> get_outs_by_unlock_block(
     const cryptonote::transaction &miner_tx,
     const std::vector<std::reference_wrapper<const cryptonote::transaction>> &txs,

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -1417,7 +1417,7 @@ void BlockchainLMDB::grow_tree(std::vector<fcmp_pp::curve_trees::OutputContext>
 
   // Use the number of leaf tuples and the existing last hashes to get a struct we can use to extend the tree
   CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set");
-  auto tree_extension = m_curve_trees->get_tree_extension(old_n_leaf_tuples, last_hashes, std::move(new_outputs));
+  auto tree_extension = m_curve_trees->get_tree_extension(old_n_leaf_tuples, last_hashes, {std::move(new_outputs)});
 
   // Insert the leaves
   // TODO: grow_leaves

src/fcmp_pp/curve_trees.cpp

@@ -868,7 +868,7 @@ template<typename C1, typename C2>
 typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extension(
     const uint64_t old_n_leaf_tuples,
     const LastHashes &existing_last_hashes,
-    std::vector<OutputContext> &&new_outputs)
+    std::vector<std::vector<OutputContext>> &&new_outputs)
 {
     TreeExtension tree_extension;
     tree_extension.leaves.start_leaf_tuple_idx = old_n_leaf_tuples;
@@ -879,16 +879,25 @@ typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extensio
     TIME_MEASURE_START(sorting_outputs);
 
     // Sort the outputs by order they appear in the chain
-    const auto sort_fn = [](const OutputContext &a, const OutputContext &b) { return a.output_id < b.output_id; };
-    std::sort(new_outputs.begin(), new_outputs.end(), sort_fn);
+    // Note: the outputs are expected to be grouped by unlock block
+    std::vector<OutputContext> flat_sorted_outputs;
+    for (auto &unsorted_outputs : new_outputs)
+    {
+        const auto sort_fn = [](const OutputContext &a, const OutputContext &b) { return a.output_id < b.output_id; };
+        std::sort(unsorted_outputs.begin(), unsorted_outputs.end(), sort_fn);
+
+        flat_sorted_outputs.insert(flat_sorted_outputs.end(),
+            std::make_move_iterator(unsorted_outputs.begin()),
+            std::make_move_iterator(unsorted_outputs.end()));
+    }
 
     TIME_MEASURE_FINISH(sorting_outputs);
 
     // Convert sorted outputs into leaf tuples, place each element of each leaf tuple in a flat vector to be hashed,
     // and place the outputs in a tree extension struct for insertion into the db. We ignore invalid outputs, since
     // they cannot be inserted to the tree.
     std::vector<typename C2::Scalar> flattened_leaves;
-    this->set_valid_leaves(flattened_leaves, tree_extension.leaves.tuples, std::move(new_outputs));
+    this->set_valid_leaves(flattened_leaves, tree_extension.leaves.tuples, std::move(flat_sorted_outputs));
 
     if (flattened_leaves.empty())
         return tree_extension;
@@ -966,7 +975,7 @@ typename CurveTrees<C1, C2>::TreeExtension CurveTrees<C1, C2>::get_tree_extensio
 template CurveTrees<Helios, Selene>::TreeExtension CurveTrees<Helios, Selene>::get_tree_extension(
     const uint64_t old_n_leaf_tuples,
     const LastHashes &existing_last_hashes,
-    std::vector<OutputContext> &&new_outputs);
+    std::vector<std::vector<OutputContext>> &&new_outputs);
 //----------------------------------------------------------------------------------------------------------------------
 template<typename C1, typename C2>
 std::vector<TrimLayerInstructions> CurveTrees<C1, C2>::get_trim_instructions(
@@ -1380,7 +1389,13 @@ void CurveTrees<C1, C2>::set_valid_leaves(
                         const auto &output_pair = new_outputs[j].output_pair;
 
                         try { pre_leaves[j] = output_to_pre_leaf_tuple(output_pair); }
-                        catch(...) { /* Invalid outputs can't be added to the tree */ return; }
+                        catch(...)
+                        {
+                            /* Invalid outputs can't be added to the tree */
+                            LOG_PRINT_L2("Output " << new_outputs[j].output_id << " is invalid (out pubkey " <<
+                                output_pair.output_pubkey << " , commitment " << output_pair.commitment << ")");
+                            continue;
+                        }
 
                         valid_outputs[j] = True;
                     }

src/fcmp_pp/curve_trees.h

@@ -305,6 +305,8 @@ class CurveTrees
             c1_layers.clear();
             c2_layers.clear();
         }
+
+        bool empty() { return leaves.empty() && c1_layers.empty() && c2_layers.empty(); }
     };
 
     // The indexes in the tree of a leaf's path elems containing whole chunks at each layer
@@ -339,7 +341,7 @@ class CurveTrees
     // outputs to add to the tree, and return a tree extension struct that can be used to extend a tree
     TreeExtension get_tree_extension(const uint64_t old_n_leaf_tuples,
         const LastHashes &existing_last_hashes,
-        std::vector<OutputContext> &&new_leaf_tuples);
+        std::vector<std::vector<OutputContext>> &&new_outputs);
 
     // Get instructions useful for trimming all existing layers in the tree
     // - always_regrow_with_remaining will use hash_grow with remaining elems left in a chunk to "trim" every chunk,