Skip to content

Commit

Permalink
migrate output positions to id-based keys
Browse files Browse the repository at this point in the history
  • Loading branch information
vertose committed May 24, 2022
1 parent 9e5a88f commit 8db5e4b
Show file tree
Hide file tree
Showing 12 changed files with 150 additions and 140 deletions.
6 changes: 3 additions & 3 deletions api/src/handlers/blocks_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ impl HeaderHandler {
)
}
};
match w(&self.chain)?.get_header_for_output(oid.commitment()) {
match w(&self.chain)?.get_header_for_output(oid.id()) {
Ok(header) => Ok(BlockHeaderPrintable::from_header(&header)),
Err(e) => Err(ErrorKind::NotFound(format!(
"Header for output {}, {}",
Expand Down Expand Up @@ -114,7 +114,7 @@ impl HeaderHandler {
return Err(ErrorKind::NotFound(format!("Output {} not found", commit)).into())
}
};
match w(&self.chain)?.get_header_for_output(oid.commitment()) {
match w(&self.chain)?.get_header_for_output(oid.id()) {
Ok(header) => return Ok(header.hash()),
Err(e) => {
return Err(ErrorKind::NotFound(format!(
Expand Down Expand Up @@ -230,7 +230,7 @@ impl BlockHandler {
Some((_, o)) => o,
None => return Err(ErrorKind::NotFound(format!("Output {}", commit)).into()),
};
match w(&self.chain)?.get_header_for_output(oid.commitment()) {
match w(&self.chain)?.get_header_for_output(oid.id()) {
Ok(header) => return Ok(header.hash()),
Err(e) => {
return Err(ErrorKind::NotFound(format!(
Expand Down
17 changes: 10 additions & 7 deletions api/src/handlers/transactions_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

use super::utils::w;
use crate::chain;
use crate::core::core::hash::Hashed;
use crate::rest::*;
use crate::router::{Handler, ResponseFuture};
use crate::types::*;
Expand Down Expand Up @@ -133,19 +134,21 @@ impl TxHashSetHandler {
let c = util::from_hex(id)
.map_err(|e| ErrorKind::Argument(format!("Not a valid commitment {}, {}", id, e)))?;
let commit = Commitment::from_vec(c);
let output_id = commit.hash();
let chain = w(&self.chain)?;
let output_pos = chain.get_output_pos(&commit).map_err(|e| {
let output_pos = chain.get_output_pos(&output_id).map_err(|e| {
ErrorKind::NotFound(format!(
"Unable to get a MMR position for commit {}, {}",
id, e
))
})?;
let merkle_proof = chain::Chain::get_merkle_proof_for_pos(&chain, commit).map_err(|e| {
ErrorKind::NotFound(format!(
"Unable to get a merkle proof for commit {}, {}",
id, e
))
})?;
let merkle_proof =
chain::Chain::get_merkle_proof_for_pos(&chain, output_id).map_err(|e| {
ErrorKind::NotFound(format!(
"Unable to get a merkle proof for commit {}, {}",
id, e
))
})?;
Ok(OutputPrintable {
output_type: OutputType::Coinbase,
commit: Commitment::from_vec(vec![]),
Expand Down
3 changes: 2 additions & 1 deletion api/src/handlers/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

use crate::chain;
use crate::chain::types::CommitPos;
use crate::core::core::hash::Hashed;
use crate::core::core::OutputIdentifier;
use crate::rest::*;
use crate::types::*;
Expand All @@ -37,7 +38,7 @@ fn get_unspent(
let c = util::from_hex(id)
.map_err(|_| ErrorKind::Argument(format!("Not a valid commitment: {}", id)))?;
let commit = Commitment::from_vec(c);
let res = chain.get_unspent(commit)?;
let res = chain.get_unspent(commit.hash())?;
Ok(res)
}

Expand Down
2 changes: 1 addition & 1 deletion api/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ impl OutputPrintable {
OutputType::Transaction
};

let pos = chain.get_unspent(output.commitment())?;
let pos = chain.get_unspent(output.id())?;

let spent = pos.is_none();

Expand Down
34 changes: 24 additions & 10 deletions chain/src/chain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,8 @@ impl Chain {
// DB migrations to be run prior to the chain being used.
// Migrate full blocks to protocol version v3.
Chain::migrate_db_v2_v3(&store)?;
// Migrate output positions from commitment-based to ID-based.
Chain::migrate_db_outputs(&store)?;

// open the txhashset, creating a new one if necessary
let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None)?;
Expand Down Expand Up @@ -711,9 +713,9 @@ impl Chain {
/// Returns Err if something went wrong beyond not finding the output.
pub fn get_unspent(
&self,
commit: Commitment,
output_id: Hash,
) -> Result<Option<(OutputIdentifier, CommitPos)>, Error> {
self.txhashset.read().get_unspent(commit)
self.txhashset.read().get_unspent(output_id)
}

/// Retrieves an unspent output using its PMMR position
Expand Down Expand Up @@ -920,9 +922,9 @@ impl Chain {

/// Return a merkle proof valid for the current output pmmr state at the
/// given pos
pub fn get_merkle_proof_for_pos(&self, commit: Commitment) -> Result<MerkleProof, Error> {
pub fn get_merkle_proof_for_pos(&self, output_id: Hash) -> Result<MerkleProof, Error> {
let mut txhashset = self.txhashset.write();
txhashset.merkle_proof(commit)
txhashset.merkle_proof(output_id)
}

/// Provides a reading view into the current txhashset state as well as
Expand Down Expand Up @@ -1412,8 +1414,8 @@ impl Chain {
}

/// Return Commit's MMR position
pub fn get_output_pos(&self, commit: &Commitment) -> Result<u64, Error> {
Ok(self.txhashset.read().get_output_pos(commit)?)
pub fn get_output_pos(&self, output_id: &Hash) -> Result<u64, Error> {
Ok(self.txhashset.read().get_output_pos(&output_id)?)
}

/// outputs by insertion index
Expand Down Expand Up @@ -1550,16 +1552,28 @@ impl Chain {
Ok(())
}

/// Migrate our local db outputs from commitment-based to ID-based.
fn migrate_db_outputs(store: &ChainStore) -> Result<(), Error> {
let batch = store.batch()?;
let migrated_count = batch.migrate_output_positions()?;
debug!(
"migrate_db_outputs: migrated {} output position entries",
migrated_count
);
batch.commit()?;
Ok(())
}

/// Gets the block header in which a given output appears in the txhashset.
pub fn get_header_for_output(&self, commit: Commitment) -> Result<BlockHeader, Error> {
pub fn get_header_for_output(&self, output_id: Hash) -> Result<BlockHeader, Error> {
let header_pmmr = self.header_pmmr.read();
let txhashset = self.txhashset.read();
let (_, pos) = match txhashset.get_unspent(commit)? {
let (_, pos) = match txhashset.get_unspent(output_id)? {
Some(o) => o,
None => {
return Err(ErrorKind::OutputNotFound(format!(
"Not found commit {}",
commit.to_hex()
"Not found output {}",
output_id.to_hex()
))
.into())
}
Expand Down
57 changes: 39 additions & 18 deletions chain/src/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ const BLOCK_PREFIX: u8 = b'b';
const HEAD_PREFIX: u8 = b'H';
const TAIL_PREFIX: u8 = b'T';
const HEADER_HEAD_PREFIX: u8 = b'G';
const OUTPUT_POS_PREFIX: u8 = b'p';
const OUTPUT_ID_POS_PREFIX: u8 = b'o';
const OUTPUT_COMMIT_POS_PREFIX: u8 = b'p'; // deprecated

/// Prefix for NRD kernel pos index lists.
pub const NRD_KERNEL_LIST_PREFIX: u8 = b'K';
Expand Down Expand Up @@ -125,19 +126,19 @@ impl ChainStore {
}

/// Get PMMR pos for the given output commitment.
pub fn get_output_pos(&self, commit: &Commitment) -> Result<u64, Error> {
match self.get_output_pos_height(commit)? {
pub fn get_output_pos(&self, output_id: &Hash) -> Result<u64, Error> {
match self.get_output_pos_height(output_id)? {
Some(pos) => Ok(pos.pos),
None => Err(Error::NotFoundErr(format!(
"Output position for: {:?}",
commit
output_id
))),
}
}

/// Get PMMR pos and block height for the given output commitment.
pub fn get_output_pos_height(&self, commit: &Commitment) -> Result<Option<CommitPos>, Error> {
self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit))
pub fn get_output_pos_height(&self, output_id: &Hash) -> Result<Option<CommitPos>, Error> {
self.db.get_ser(&to_key(OUTPUT_ID_POS_PREFIX, output_id))
}

/// Builds a new batch to be used with this store.
Expand Down Expand Up @@ -270,6 +271,26 @@ impl<'a> Batch<'a> {
Ok(())
}

/// Migrate output positions from commitment-based to ID-based keys.
pub fn migrate_output_positions(&self) -> Result<usize, Error> {
let start_key = to_key(OUTPUT_COMMIT_POS_PREFIX, "");

let mut migrated_count = 0;
let commit_pos_iter: SerIterator<(u64, u64)> = self.db.iter(&start_key)?;
for (key, (pos, height)) in commit_pos_iter {
// Recover commitment from key, which is in format 'p:commitment'
let commit = Commitment::from_vec(key[2..].to_vec());

// Save output position in new format
self.save_output_pos_height(&commit.hash(), CommitPos { pos, height })?;

// Delete the old entry
self.db.delete(&key)?;
migrated_count += 1;
}
Ok(migrated_count)
}

/// Low level function to delete directly by raw key.
pub fn delete(&self, key: &[u8]) -> Result<(), Error> {
self.db.delete(key)
Expand Down Expand Up @@ -321,14 +342,14 @@ impl<'a> Batch<'a> {
}

/// Save output_pos and block height to index.
pub fn save_output_pos_height(&self, commit: &Commitment, pos: CommitPos) -> Result<(), Error> {
pub fn save_output_pos_height(&self, output_id: &Hash, pos: CommitPos) -> Result<(), Error> {
self.db
.put_ser(&to_key(OUTPUT_POS_PREFIX, commit)[..], &pos)
.put_ser(&to_key(OUTPUT_ID_POS_PREFIX, output_id)[..], &pos)
}

/// Delete the output_pos index entry for a spent output.
pub fn delete_output_pos_height(&self, commit: &Commitment) -> Result<(), Error> {
self.db.delete(&to_key(OUTPUT_POS_PREFIX, commit))
pub fn delete_output_pos_height(&self, output_id: &Hash) -> Result<(), Error> {
self.db.delete(&to_key(OUTPUT_ID_POS_PREFIX, output_id))
}

/// Delete the commitment for a spent output.
Expand Down Expand Up @@ -356,31 +377,31 @@ impl<'a> Batch<'a> {
/// When using the output_pos iterator we have access to the index keys but not the
/// original commitment that the key is constructed from. So we need a way of comparing
/// a key with another commitment without reconstructing the commitment from the key bytes.
pub fn is_match_output_pos_key(&self, key: &[u8], commit: &Commitment) -> bool {
let commit_key = to_key(OUTPUT_POS_PREFIX, commit);
pub fn is_match_output_pos_key(&self, key: &[u8], output_id: &Hash) -> bool {
let commit_key = to_key(OUTPUT_ID_POS_PREFIX, output_id);
commit_key == key
}

/// Iterator over the output_pos index.
pub fn output_pos_iter(&self) -> Result<SerIterator<(u64, u64)>, Error> {
let key = to_key(OUTPUT_POS_PREFIX, "");
let key = to_key(OUTPUT_ID_POS_PREFIX, "");
self.db.iter(&key)
}

/// Get output_pos from index.
pub fn get_output_pos(&self, commit: &Commitment) -> Result<u64, Error> {
match self.get_output_pos_height(commit)? {
pub fn get_output_pos(&self, output_id: &Hash) -> Result<u64, Error> {
match self.get_output_pos_height(output_id)? {
Some(pos) => Ok(pos.pos),
None => Err(Error::NotFoundErr(format!(
"Output position for: {:?}",
commit
output_id
))),
}
}

/// Get output_pos and block height from index.
pub fn get_output_pos_height(&self, commit: &Commitment) -> Result<Option<CommitPos>, Error> {
self.db.get_ser(&to_key(OUTPUT_POS_PREFIX, commit))
pub fn get_output_pos_height(&self, output_id: &Hash) -> Result<Option<CommitPos>, Error> {
self.db.get_ser(&to_key(OUTPUT_ID_POS_PREFIX, output_id))
}

/// Get the previous header.
Expand Down
Loading

0 comments on commit 8db5e4b

Please sign in to comment.