Skip to content

Commit

Permalink
fcmp++ TreeSync: keep track of locked outputs, grow as they unlock
Browse files Browse the repository at this point in the history
- Need to keep a locked outputs cache in the TreeSync object so
that we can grow the tree correctly, the same way as the daemon.
- On reorg, we make sure to delete all outputs from the cache
that were created at the popped block.
- We keep the locked outputs cached for N blocks, where N is max
reorg depth, so that upon popping a block, we retain all outputs
trimmed from the tree but still in the chain that would eventually
be used to grow the tree again upon unlock.
- Added a function to clear the TreeSyncMemory object state for
wallet2.
  • Loading branch information
j-berman committed Oct 18, 2024
1 parent d29e1e2 commit 178c39d
Show file tree
Hide file tree
Showing 15 changed files with 616 additions and 281 deletions.
12 changes: 12 additions & 0 deletions src/blockchain_db/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

monero_add_library(blockchain_db_utils
blockchain_db_utils.cpp
)

target_link_libraries(blockchain_db_utils
PUBLIC
cncrypto
cryptonote_basic
ringct
)

set(blockchain_db_sources
blockchain_db.cpp
lmdb/db_lmdb.cpp
Expand All @@ -43,6 +54,7 @@ monero_add_library(blockchain_db
${blockchain_db_private_headers})
target_link_libraries(blockchain_db
PUBLIC
blockchain_db_utils
common
cncrypto
fcmp_pp
Expand Down
106 changes: 12 additions & 94 deletions src/blockchain_db/blockchain_db.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

#include "string_tools.h"
#include "blockchain_db.h"
#include "blockchain_db_utils.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "profile_tools.h"
#include "ringct/rctOps.h"
Expand All @@ -41,60 +42,6 @@

using epee::string_tools::pod_to_hex;

//---------------------------------------------------------------
// Helper function to group outputs by unlock block
static void get_outs_by_unlock_block(const cryptonote::transaction &tx,
const std::vector<uint64_t> &output_ids,
const uint64_t tx_height,
const bool miner_tx,
fcmp_pp::curve_trees::OutputsByUnlockBlock &outs_by_unlock_block_inout)
{
const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, tx_height);

CHECK_AND_ASSERT_THROW_MES(tx.vout.size() == output_ids.size(), "unexpected size of output ids");

for (std::size_t i = 0; i < tx.vout.size(); ++i)
{
const auto &out = tx.vout[i];

crypto::public_key output_public_key;
if (!cryptonote::get_output_public_key(out, output_public_key))
throw std::runtime_error("Could not get an output public key from a tx output.");

static_assert(CURRENT_TRANSACTION_VERSION == 2, "This section of code was written with 2 tx versions in mind. "
"Revisit this section and update for the new tx version.");
CHECK_AND_ASSERT_THROW_MES(tx.version == 1 || tx.version == 2, "encountered unexpected tx version");

if (!miner_tx && tx.version == 2)
CHECK_AND_ASSERT_THROW_MES(tx.rct_signatures.outPk.size() > i, "unexpected size of outPk");

rct::key commitment = (miner_tx || tx.version != 2)
? rct::zeroCommit(out.amount)
: tx.rct_signatures.outPk[i].mask;

auto output_pair = fcmp_pp::curve_trees::OutputPair{
.output_pubkey = std::move(output_public_key),
.commitment = std::move(commitment)
};

auto output_context = fcmp_pp::curve_trees::OutputContext{
.output_id = output_ids[i],
.output_pair = std::move(output_pair)
};

if (outs_by_unlock_block_inout.find(unlock_block) == outs_by_unlock_block_inout.end())
{
auto new_vec = std::vector<fcmp_pp::curve_trees::OutputContext>{std::move(output_context)};
outs_by_unlock_block_inout[unlock_block] = std::move(new_vec);
}
else
{
outs_by_unlock_block_inout[unlock_block].emplace_back(std::move(output_context));
}
}
}
//---------------------------------------------------------------

namespace cryptonote
{

Expand Down Expand Up @@ -233,7 +180,7 @@ void BlockchainDB::pop_block()
pop_block(blk, txs);
}

std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
{
const transaction &tx = txp.first;

Expand Down Expand Up @@ -277,7 +224,7 @@ std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash

uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash);

std::vector<output_indexes_t> output_indices(tx.vout.size());
std::vector<uint64_t> amount_output_indices(tx.vout.size());

// iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index
Expand All @@ -291,29 +238,17 @@ std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash
cryptonote::tx_out vout = tx.vout[i];
rct::key commitment = rct::zeroCommit(vout.amount);
vout.amount = 0;
output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time,
amount_output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time,
&commitment);
}
else
{
output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL);
}
}

std::vector<uint64_t> amount_output_indices;
std::vector<uint64_t> output_ids;
amount_output_indices.reserve(output_indices.size());
output_ids.reserve(output_indices.size());
for (const auto &o_idx : output_indices)
{
amount_output_indices.push_back(o_idx.amount_index);
output_ids.push_back(o_idx.output_id);
}

add_tx_amount_output_indices(tx_id, amount_output_indices);

return output_ids;
}

uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
Expand All @@ -336,57 +271,40 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
time_blk_hash += time1;

uint64_t prev_height = height();
const uint64_t total_n_outputs = num_outputs();

// call out to add the transactions

time1 = epee::misc_utils::get_tick_count();

std::vector<std::vector<uint64_t>> output_ids;
output_ids.reserve(txs.size());

uint64_t num_rct_outs = 0;
blobdata miner_bd = tx_to_blob(blk.miner_tx);
std::vector<uint64_t> miner_output_ids = add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
if (blk.miner_tx.version == 2)
num_rct_outs += blk.miner_tx.vout.size();
int tx_i = 0;
crypto::hash tx_hash = crypto::null_hash;
std::vector<std::reference_wrapper<const transaction>> _txs; // ref wrapper to avoid extra copies
_txs.reserve(txs.size());
for (const std::pair<transaction, blobdata>& tx : txs)
{
tx_hash = blk.tx_hashes[tx_i];
output_ids.push_back(add_transaction(blk_hash, tx, &tx_hash));
add_transaction(blk_hash, tx, &tx_hash);
for (const auto &vout: tx.first.vout)
{
if (vout.amount == 0)
++num_rct_outs;
}
++tx_i;
_txs.push_back(std::ref(tx.first));
}
TIME_MEASURE_FINISH(time1);
time_add_transaction += time1;

// When adding a block, we also need to keep track of when outputs unlock, so
// we can use them to grow the merkle tree used in fcmp's at that point.
fcmp_pp::curve_trees::OutputsByUnlockBlock outs_by_unlock_block;

// Get miner tx's leaf tuples
get_outs_by_unlock_block(
blk.miner_tx,
miner_output_ids,
prev_height,
true/*miner_tx*/,
outs_by_unlock_block);

// Get all other txs' leaf tuples
for (std::size_t i = 0; i < txs.size(); ++i)
{
get_outs_by_unlock_block(
txs[i].first,
output_ids[i],
prev_height,
false/*miner_tx*/,
outs_by_unlock_block);
}
cryptonote::get_outs_by_unlock_block(blk.miner_tx, _txs, total_n_outputs, prev_height, outs_by_unlock_block);

// call out to subclass implementation to add the block & metadata
time1 = epee::misc_utils::get_tick_count();
Expand Down
27 changes: 13 additions & 14 deletions src/blockchain_db/blockchain_db.h
Original file line number Diff line number Diff line change
Expand Up @@ -190,15 +190,6 @@ struct txpool_tx_meta_t
}
};

/**
* @brief a struct containing output indexes for convenience
*/
struct output_indexes_t
{
uint64_t amount_index;
uint64_t output_id;
};

#define DBF_SAFE 1
#define DBF_FAST 2
#define DBF_FASTEST 4
Expand Down Expand Up @@ -498,9 +489,9 @@ class BlockchainDB
* @param local_index index of the output in its transaction
* @param unlock_time unlock time/height of the output
* @param commitment the rct commitment to the output amount
* @return output indexes
* @return amount output index
*/
virtual output_indexes_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;

/**
* @brief store amount output indices for a tx's outputs
Expand Down Expand Up @@ -581,10 +572,8 @@ class BlockchainDB
* @param tx the transaction to add
* @param tx_hash_ptr the hash of the transaction, if already calculated
* @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated
*
* @return the global output ids of all outputs inserted
*/
std::vector<uint64_t> add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);

mutable uint64_t time_tx_exists = 0; //!< a performance metric
uint64_t time_commit1 = 0; //!< a performance metric
Expand Down Expand Up @@ -1395,6 +1384,16 @@ class BlockchainDB
*/
virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0;

// returns the total number of outputs in the chain (of all amounts)
/**
* @brief fetches the number of outputs in the chain
*
* The subclass should return a count of outputs, or zero if there are none.
*
* @return the total number of outputs in the chain (of all amounts)
*/
virtual uint64_t num_outputs() const = 0;

// returns the total number of outputs of amount <amount>
/**
* @brief fetches the number of outputs of a given amount
Expand Down
124 changes: 124 additions & 0 deletions src/blockchain_db/blockchain_db_utils.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright (c) 2024, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "blockchain_db_utils.h"

#include "crypto/crypto.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "ringct/rctOps.h"

//----------------------------------------------------------------------------------------------------------------------
// Helper function to group outputs by unlock block
static uint64_t set_tx_outs_by_unlock_block(const cryptonote::transaction &tx,
const uint64_t &first_output_id,
const uint64_t block_idx,
const bool miner_tx,
fcmp_pp::curve_trees::OutputsByUnlockBlock &outs_by_unlock_block_inout)
{
const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, block_idx);

for (std::size_t i = 0; i < tx.vout.size(); ++i)
{
const auto &out = tx.vout[i];

crypto::public_key output_public_key;
if (!cryptonote::get_output_public_key(out, output_public_key))
throw std::runtime_error("Could not get an output public key from a tx output.");

static_assert(CURRENT_TRANSACTION_VERSION == 2, "This section of code was written with 2 tx versions in mind. "
"Revisit this section and update for the new tx version.");
CHECK_AND_ASSERT_THROW_MES(tx.version == 1 || tx.version == 2, "encountered unexpected tx version");

if (!miner_tx && tx.version == 2)
CHECK_AND_ASSERT_THROW_MES(tx.rct_signatures.outPk.size() > i, "unexpected size of outPk");

rct::key commitment = (miner_tx || tx.version < 2)
? rct::zeroCommit(out.amount)
: tx.rct_signatures.outPk[i].mask;

auto output_pair = fcmp_pp::curve_trees::OutputPair{
.output_pubkey = std::move(output_public_key),
.commitment = std::move(commitment)
};

auto output_context = fcmp_pp::curve_trees::OutputContext{
.output_id = first_output_id + i,
.output_pair = std::move(output_pair)
};

if (outs_by_unlock_block_inout.find(unlock_block) == outs_by_unlock_block_inout.end())
{
auto new_vec = std::vector<fcmp_pp::curve_trees::OutputContext>{std::move(output_context)};
outs_by_unlock_block_inout[unlock_block] = std::move(new_vec);
}
else
{
outs_by_unlock_block_inout[unlock_block].emplace_back(std::move(output_context));
}
}

return tx.vout.size();
}
//----------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------
namespace cryptonote
{
uint64_t get_outs_by_unlock_block(
const cryptonote::transaction &miner_tx,
const std::vector<std::reference_wrapper<const cryptonote::transaction>> &txs,
const uint64_t first_output_id,
const uint64_t block_idx,
fcmp_pp::curve_trees::OutputsByUnlockBlock &outs_by_unlock_block_out)
{
outs_by_unlock_block_out.clear();

uint64_t output_id = first_output_id;

// Get miner tx's leaf tuples
output_id += set_tx_outs_by_unlock_block(
miner_tx,
output_id,
block_idx,
true/*miner_tx*/,
outs_by_unlock_block_out);

// Get all other txs' leaf tuples
for (const auto &tx : txs)
{
output_id += set_tx_outs_by_unlock_block(
tx.get(),
output_id,
block_idx,
false/*miner_tx*/,
outs_by_unlock_block_out);
}

return output_id;
}
//----------------------------------------------------------------------------------------------------------------------
}//namespace cryptonote
Loading

0 comments on commit 178c39d

Please sign in to comment.