From ed496e96b164e209d17e26316f3880aa87820fe5 Mon Sep 17 00:00:00 2001 From: Justin Moon Date: Mon, 8 Jun 2020 12:59:24 -0500 Subject: [PATCH 1/4] WIP output script descriptor support --- src/app.rs | 14 +-- src/config.rs | 20 ++++ src/hd.rs | 266 ++++++++++++++++++-------------------------------- src/http.rs | 55 ++++++----- src/query.rs | 30 +++--- src/types.rs | 36 +++++++ 6 files changed, 203 insertions(+), 218 deletions(-) diff --git a/src/app.rs b/src/app.rs index ca94ca9..68be2ee 100644 --- a/src/app.rs +++ b/src/app.rs @@ -36,19 +36,19 @@ impl App { pub fn boot(config: Config) -> Result { debug!("{:?}", config); - let wallets = HDWallet::from_xpubs( - &config.xpubs[..], - &config.bare_xpubs[..], + let rpc = RpcClient::new(config.bitcoind_url(), config.bitcoind_auth()?)?; + + let wallets = HDWallet::from_descriptors( + &config.descriptors[..], config.network, config.gap_limit, config.initial_import_size, + &rpc, )?; + + let rpc = Arc::new(rpc); let watcher = HDWatcher::new(wallets); - let rpc = Arc::new(RpcClient::new( - config.bitcoind_url(), - config.bitcoind_auth()?, - )?); let indexer = Arc::new(RwLock::new(Indexer::new(rpc.clone(), watcher))); let query = Arc::new(Query::new(config.network, rpc.clone(), indexer.clone())); diff --git a/src/config.rs b/src/config.rs index b8782d3..84d6551 100644 --- a/src/config.rs +++ b/src/config.rs @@ -96,6 +96,16 @@ pub struct Config { )] pub bitcoind_cookie: Option, + #[structopt( + short = "D", + long = "descriptors", + help = "descriptors to track and since when (rescans from genesis by default, use : or : to specify a timestmap, or :none to disable rescan)", + parse(try_from_str = parse_descriptor), + env, hide_env_values(true), use_delimiter(true), + display_order(20) + )] + pub descriptors: Vec<(String, RescanSince)>, + #[structopt( short = "x", long = "xpub", @@ -346,6 +356,16 @@ fn parse_xpub(s: &str) -> Result<(XyzPubKey, RescanSince)> { Ok((xpub, rescan)) } +fn parse_descriptor(s: &str) -> Result<(String, RescanSince)> { + let mut parts = s.splitn(2, ':'); + let descriptor = String::from(parts.next().unwrap()); + let rescan = parts + .next() + .map_or(Ok(RescanSince::Timestamp(0)), parse_rescan) + .unwrap(); + Ok((descriptor, rescan)) +} + fn parse_rescan(s: &str) -> Result { Ok(match s { "none" => RescanSince::Now, diff --git a/src/hd.rs b/src/hd.rs index a9ebb97..96bcf63 100644 --- a/src/hd.rs +++ b/src/hd.rs @@ -5,13 +5,13 @@ use std::str::FromStr; use serde::Serialize; use bitcoin::secp256k1::{self, Secp256k1}; -use bitcoin::util::bip32::{ChildNumber, ExtendedPubKey, Fingerprint}; +use bitcoin::util::bip32::ExtendedPubKey; use bitcoin::{util::base58, Address, Network}; -use bitcoincore_rpc::json::{ImportMultiRequest, ImportMultiRequestScriptPubkey}; +use bitcoincore_rpc::json::ImportMultiRequest; use bitcoincore_rpc::{self as rpc, Client as RpcClient, RpcApi}; use crate::error::{Context, Result}; -use crate::types::{RescanSince, ScriptType}; +use crate::types::{Descriptor, DescriptorChecksum, RescanSince, ScriptType}; const LABEL_PREFIX: &str = "bwt"; @@ -21,7 +21,7 @@ lazy_static! { #[derive(Debug)] pub struct HDWatcher { - wallets: HashMap, + wallets: HashMap, } impl HDWatcher { @@ -29,25 +29,23 @@ impl HDWatcher { HDWatcher { wallets: wallets .into_iter() - .map(|wallet| (wallet.master.fingerprint(), wallet)) + .map(|wallet| (wallet.descriptor.checksum(), wallet)) .collect(), } - // XXX indexing by the fingerprint prevents using the same underlying public key with - // different script types ([xyz]pub), they will have the same fingerprint and conflict. } - pub fn wallets(&self) -> &HashMap { + pub fn wallets(&self) -> &HashMap { &self.wallets } - pub fn get(&self, fingerprint: Fingerprint) -> Option<&HDWallet> { - self.wallets.get(&fingerprint) + pub fn get(&self, desc_check: DescriptorChecksum) -> Option<&HDWallet> { + self.wallets.get(&desc_check) } // Mark an address as funded pub fn mark_funded(&mut self, origin: &KeyOrigin) { - if let KeyOrigin::Derived(parent_fingerprint, index) = origin { - if let Some(wallet) = self.wallets.get_mut(parent_fingerprint) { + if let KeyOrigin::Derived(desc_check, index) = origin { + if let Some(wallet) = self.wallets.get_mut(desc_check) { if wallet.max_imported_index.map_or(true, |max| *index > max) { wallet.max_imported_index = Some(*index); } @@ -63,25 +61,25 @@ impl HDWatcher { pub fn check_imports(&mut self, rpc: &RpcClient) -> Result<()> { debug!("checking previous imports"); let labels: Vec = rpc.call("listlabels", &[]).map_err(labels_error)?; - let mut imported_indexes: HashMap = HashMap::new(); + let mut imported_indexes: HashMap = HashMap::new(); for label in labels { - if let Some(KeyOrigin::Derived(fingerprint, index)) = KeyOrigin::from_label(&label) { - if self.wallets.contains_key(&fingerprint) { + if let Some(KeyOrigin::Derived(desc_check, index)) = KeyOrigin::from_label(&label) { + if self.wallets.contains_key(&desc_check) { imported_indexes - .entry(fingerprint) + .entry(desc_check) .and_modify(|current| *current = (*current).max(index)) .or_insert(index); } } } - for (fingerprint, max_imported_index) in imported_indexes { + for (desc_check, max_imported_index) in imported_indexes { trace!( "wallet {} was imported up to index {}", - fingerprint, + desc_check, max_imported_index ); - let wallet = self.wallets.get_mut(&fingerprint).unwrap(); + let wallet = self.wallets.get_mut(&desc_check).unwrap(); wallet.max_imported_index = Some(max_imported_index); // if anything was imported at all, assume we've finished the initial sync. this might @@ -96,7 +94,7 @@ impl HDWatcher { let mut import_reqs = vec![]; let mut pending_updates = vec![]; - for (fingerprint, wallet) in self.wallets.iter_mut() { + for (desc_check, wallet) in self.wallets.iter_mut() { let watch_index = wallet.watch_index(); if wallet.max_imported_index.map_or(true, |i| watch_index > i) { let start_index = wallet @@ -105,21 +103,21 @@ impl HDWatcher { debug!( "importing {} range {}-{} with rescan={}", - fingerprint, start_index, watch_index, rescan, + desc_check, start_index, watch_index, rescan, ); import_reqs.append(&mut wallet.make_imports(start_index, watch_index, rescan)); - pending_updates.push((wallet, fingerprint, watch_index)); + pending_updates.push((wallet, desc_check, watch_index)); } else if !wallet.done_initial_import { debug!( "done initial import for {} up to index {}", - fingerprint, + desc_check, wallet.max_imported_index.unwrap() ); wallet.done_initial_import = true; } else { - trace!("no imports needed for {}", fingerprint); + trace!("no imports needed for {}", desc_check); } } @@ -135,8 +133,8 @@ impl HDWatcher { info!("done importing batch"); } - for (wallet, fingerprint, imported_index) in pending_updates { - debug!("imported {} up to index {}", fingerprint, imported_index); + for (wallet, desc_check, imported_index) in pending_updates { + debug!("imported {} up to index {}", desc_check, imported_index); wallet.max_imported_index = Some(imported_index); } @@ -146,9 +144,8 @@ impl HDWatcher { #[derive(Debug, Clone)] pub struct HDWallet { - master: ExtendedPubKey, + descriptor: Descriptor, network: Network, - script_type: ScriptType, gap_limit: u32, initial_import_size: u32, rescan_policy: RescanSince, @@ -160,17 +157,15 @@ pub struct HDWallet { impl HDWallet { pub fn new( - master: ExtendedPubKey, + descriptor: Descriptor, network: Network, - script_type: ScriptType, gap_limit: u32, initial_import_size: u32, rescan_policy: RescanSince, ) -> Self { Self { - master, + descriptor, network, - script_type, gap_limit, // setting initial_import_size < gap_limit makes no sense, the user probably meant to increase both initial_import_size: initial_import_size.max(gap_limit), @@ -181,111 +176,59 @@ impl HDWallet { } } - pub fn from_bare_xpub( - xpub: XyzPubKey, - network: Network, - gap_limit: u32, - initial_import_size: u32, - rescan_policy: RescanSince, - ) -> Result { - ensure!( - xpub.matches_network(network), - "xpub network mismatch, {} is {} and not {}", - xpub, - xpub.network, - network - ); - Ok(Self::new( - xpub.extended_pubkey, - network, - xpub.script_type, - gap_limit, - initial_import_size, - rescan_policy, - )) - } - - pub fn from_xpub( - xpub: XyzPubKey, - network: Network, - gap_limit: u32, - initial_import_size: u32, - rescan_policy: RescanSince, - ) -> Result> { - ensure!( - xpub.matches_network(network), - "xpub network mismatch, {} is {} and not {}", - xpub, - xpub.network, - network - ); - Ok(vec![ - // external chain (receive) - Self::new( - xpub.extended_pubkey - .derive_pub(&*EC, &[ChildNumber::from(0)])?, - network, - xpub.script_type, - gap_limit, - initial_import_size, - rescan_policy, - ), - // internal chain (change) - Self::new( - xpub.extended_pubkey - .derive_pub(&*EC, &[ChildNumber::from(1)])?, - network, - xpub.script_type, - gap_limit, - initial_import_size, - rescan_policy, - ), - ]) - } - - pub fn from_xpubs( - xpubs: &[(XyzPubKey, RescanSince)], - bare_xpubs: &[(XyzPubKey, RescanSince)], + pub fn from_descriptors( + descriptors: &[(String, RescanSince)], network: Network, gap_limit: u32, initial_import_size: u32, + rpc: &RpcClient, ) -> Result> { let mut wallets = vec![]; - for (xpub, rescan) in xpubs { - wallets.append( - &mut Self::from_xpub( - xpub.clone(), - network, - gap_limit, - initial_import_size, - *rescan, - ) - .with_context(|| format!("invalid xpub {}", xpub))?, - ); - } - for (xpub, rescan) in bare_xpubs { + for (descriptor, rescan) in descriptors { wallets.push( - Self::from_bare_xpub( - xpub.clone(), + Self::from_descriptor( + descriptor.clone(), network, gap_limit, initial_import_size, *rescan, + rpc, ) - .with_context(|| format!("invalid xpub {}", xpub))?, + .with_context(|| format!("invalid descriptor {}", descriptor))? + .clone(), ); } if wallets.is_empty() { - error!("Please provide at least one xpub to track (via --xpub or --bare-xpub)."); - bail!("no xpubs provided"); + warn!("Please provide at least one descriptor to track (via --descriptors)."); + bail!("No descriptors provided"); } Ok(wallets) } - pub fn derive(&self, index: u32) -> ExtendedPubKey { - self.master - .derive_pub(&*EC, &[ChildNumber::from(index)]) - .unwrap() + pub fn from_descriptor( + descriptor: String, + network: Network, + gap_limit: u32, + initial_import_size: u32, + rescan_policy: RescanSince, + rpc: &RpcClient, + ) -> Result { + let descriptor = Descriptor::new(&descriptor, rpc).unwrap(); + // FIXME + //ensure!( + //xpub.matches_network(network), + //"xpub network mismatch, {} is {} and not {}", + //xpub, + //xpub.network, + //network + //); + Ok(Self::new( + descriptor, + network, + gap_limit, + initial_import_size, + rescan_policy, + )) } /// Returns the maximum index that needs to be watched @@ -305,7 +248,7 @@ impl HDWallet { start_index: u32, end_index: u32, rescan: bool, - ) -> Vec<(Address, RescanSince, String)> { + ) -> Vec<(Descriptor, u32, String, RescanSince)> { let rescan_since = if rescan { self.rescan_policy } else { @@ -314,24 +257,15 @@ impl HDWallet { (start_index..=end_index) .map(|index| { - let key = self.derive(index); - let address = self.to_address(&key); - let origin = KeyOrigin::Derived(key.parent_fingerprint, index); - (address, rescan_since, origin.to_label()) + let label = format!("{}/{}", self.descriptor.checksum(), index); + (self.descriptor.clone(), index, label, rescan_since) }) .collect() } - pub fn to_address(&self, key: &ExtendedPubKey) -> Address { - match self.script_type { - ScriptType::P2pkh => Address::p2pkh(&key.public_key, self.network), - ScriptType::P2wpkh => Address::p2wpkh(&key.public_key, self.network), - ScriptType::P2shP2wpkh => Address::p2shwpkh(&key.public_key, self.network), - } - } - - pub fn derive_address(&self, index: u32) -> Address { - self.to_address(&self.derive(index)) + pub fn derive_address(&self, index: u32, rpc: &RpcClient) -> Result
{ + let res = rpc.derive_addresses(&self.descriptor.0, Some([index, index]))?; + Ok(res[0].to_owned()) } pub fn get_next_index(&self) -> u32 { @@ -339,24 +273,23 @@ impl HDWallet { .map_or(0, |max_funded_index| max_funded_index + 1) } } -fn batch_import(rpc: &RpcClient, import_reqs: Vec<(Address, RescanSince, String)>) -> Result<()> { - // XXX use importmulti with ranged descriptors? the key derivation info won't be - // directly available on `listtransactions` and would require an additional rpc all. - +fn batch_import( + rpc: &RpcClient, + import_reqs: Vec<(Descriptor, u32, String, RescanSince)>, +) -> Result<()> { let results = rpc.import_multi( &import_reqs .iter() - .map(|(address, rescan, label)| { - trace!("importing {} as {}", address, label,); - - ImportMultiRequest { + .map( + |(descriptor, index, label, rescan_since)| ImportMultiRequest { label: Some(&label), watchonly: Some(true), - timestamp: *rescan, - script_pubkey: Some(ImportMultiRequestScriptPubkey::Address(&address)), + timestamp: *rescan_since, + range: Some((*index as usize, *index as usize)), + descriptor: Some(&descriptor.0), ..Default::default() - } - }) + }, + ) .collect::>(), None, )?; @@ -375,10 +308,10 @@ fn batch_import(rpc: &RpcClient, import_reqs: Vec<(Address, RescanSince, String) #[derive(Debug, Clone, PartialEq)] pub enum KeyOrigin { - Derived(Fingerprint, u32), + Derived(DescriptorChecksum, u32), Standalone, // bwt never does hardended derivation itself, but can receive an hardend --bare-xpub - DerivedHard(Fingerprint, u32), + DerivedHard(DescriptorChecksum, u32), } impl_string_serializer!( @@ -386,11 +319,11 @@ impl_string_serializer!( origin, match origin { KeyOrigin::Standalone => "standalone".into(), - KeyOrigin::Derived(parent_fingerprint, index) => { - format!("{}/{}", parent_fingerprint, index) + KeyOrigin::Derived(parent_desc_check, index) => { + format!("{}/{}", parent_desc_check, index) } - KeyOrigin::DerivedHard(parent_fingerprint, index) => { - format!("{}/{}'", parent_fingerprint, index) + KeyOrigin::DerivedHard(parent_desc_check, index) => { + format!("{}/{}'", parent_desc_check, index) } } ); @@ -398,9 +331,12 @@ impl_string_serializer!( impl KeyOrigin { pub fn to_label(&self) -> String { match self { - KeyOrigin::Derived(parent_fingerprint, index) => { - format!("{}/{}/{}", LABEL_PREFIX, parent_fingerprint, index) - } + KeyOrigin::Derived(parent_desc_check, index) => format!( + "{}/{}/{}", + LABEL_PREFIX, + hex::encode(parent_desc_check.0.as_bytes()), + index + ), KeyOrigin::Standalone => LABEL_PREFIX.into(), KeyOrigin::DerivedHard(..) => unreachable!(), } @@ -409,8 +345,8 @@ impl KeyOrigin { pub fn from_label(s: &str) -> Option { let parts: Vec<&str> = s.splitn(3, '/').collect(); match (parts.get(0), parts.get(1), parts.get(2)) { - (Some(&LABEL_PREFIX), Some(parent), Some(index)) => Some(KeyOrigin::Derived( - Fingerprint::from_str(parent).ok()?, + (Some(parent), Some(index), None) => Some(KeyOrigin::Derived( + DescriptorChecksum(parent.to_string()), index.parse().ok()?, )), (Some(&LABEL_PREFIX), None, None) => Some(KeyOrigin::Standalone), @@ -418,18 +354,6 @@ impl KeyOrigin { } } - pub fn from_extkey(key: &ExtendedPubKey) -> Self { - let parent = key.parent_fingerprint; - if parent[..] == [0, 0, 0, 0] { - KeyOrigin::Standalone - } else { - match key.child_number { - ChildNumber::Normal { index } => KeyOrigin::Derived(parent, index), - ChildNumber::Hardened { index } => KeyOrigin::DerivedHard(parent, index), - } - } - } - pub fn is_standalone(origin: &KeyOrigin) -> bool { match origin { KeyOrigin::Standalone => true, @@ -526,10 +450,8 @@ impl Serialize for HDWallet { S: serde::Serializer, { let mut rgb = serializer.serialize_struct("HDWallet", 3)?; - rgb.serialize_field("xpub", &self.master)?; - rgb.serialize_field("origin", &KeyOrigin::from_extkey(&self.master))?; + rgb.serialize_field("descriptor", &self.descriptor.0)?; rgb.serialize_field("network", &self.network)?; - rgb.serialize_field("script_type", &self.script_type)?; rgb.serialize_field("gap_limit", &self.gap_limit)?; rgb.serialize_field("initial_import_size", &self.initial_import_size)?; rgb.serialize_field("rescan_policy", &self.rescan_policy)?; diff --git a/src/http.rs b/src/http.rs index 5d63a72..0d4f0b5 100644 --- a/src/http.rs +++ b/src/http.rs @@ -9,12 +9,11 @@ use warp::http::{header, StatusCode}; use warp::sse::ServerSentEvent; use warp::{reply, Filter, Reply}; -use bitcoin::util::bip32::Fingerprint; use bitcoin::{Address, BlockHash, OutPoint, Txid}; use bitcoin_hashes::hex::FromHex; use crate::error::{fmt_error_chain, BwtError, Error, OptionExt}; -use crate::types::{BlockId, ScriptHash}; +use crate::types::{BlockId, DescriptorChecksum, ScriptHash}; use crate::{store, IndexChange, Query}; type SyncChanSender = Arc>>; @@ -53,11 +52,11 @@ async fn run( // GET /hd/:fingerprint let hd_wallet_handler = warp::get() - .and(warp::path!("hd" / Fingerprint)) + .and(warp::path!("hd" / DescriptorChecksum)) .and(query.clone()) - .map(|fingerprint: Fingerprint, query: Arc| { + .map(|desc_check: DescriptorChecksum, query: Arc| { let wallet = query - .get_hd_wallet(fingerprint) + .get_hd_wallet(desc_check) .or_err(StatusCode::NOT_FOUND)?; Ok(reply::json(&wallet)) }) @@ -65,23 +64,25 @@ async fn run( // GET /hd/:fingerprint/:index let hd_key_handler = warp::get() - .and(warp::path!("hd" / Fingerprint / u32)) + .and(warp::path!("hd" / DescriptorChecksum / u32)) .and(query.clone()) - .map(|fingerprint: Fingerprint, index: u32, query: Arc| { - let script_info = query - .get_hd_script_info(fingerprint, index) - .or_err(StatusCode::NOT_FOUND)?; - Ok(reply::json(&script_info)) - }) + .map( + |desc_check: DescriptorChecksum, index: u32, query: Arc| { + let script_info = query + .get_hd_script_info(desc_check, index) + .or_err(StatusCode::NOT_FOUND)?; + Ok(reply::json(&script_info)) + }, + ) .map(handle_error); // GET /hd/:fingerprint/gap let hd_gap_handler = warp::get() - .and(warp::path!("hd" / Fingerprint / "gap")) + .and(warp::path!("hd" / DescriptorChecksum / "gap")) .and(query.clone()) - .map(|fingerprint: Fingerprint, query: Arc| { + .map(|desc_check: DescriptorChecksum, query: Arc| { let gap = query - .find_hd_gap(fingerprint) + .find_hd_gap(desc_check) .or_err(StatusCode::NOT_FOUND)?; Ok(reply::json(&gap)) }) @@ -89,14 +90,14 @@ async fn run( // GET /hd/:fingerprint/next let hd_next_handler = warp::get() - .and(warp::path!("hd" / Fingerprint / "next")) + .and(warp::path!("hd" / DescriptorChecksum / "next")) .and(query.clone()) - .map(|fingerprint: Fingerprint, query: Arc| { + .map(|desc_check: DescriptorChecksum, query: Arc| { let wallet = query - .get_hd_wallet(fingerprint) + .get_hd_wallet(desc_check.clone()) .or_err(StatusCode::NOT_FOUND)?; let next_index = wallet.get_next_index(); - let uri = format!("/hd/{}/{}", fingerprint, next_index); + let uri = format!("/hd/{}/{}", desc_check.clone(), next_index); // issue a 307 redirect to the hdkey resource uri, and also include the derivation // index in the response Ok(reply::with_header( @@ -115,14 +116,16 @@ async fn run( // TODO check address version bytes matches the configured network // GET /hd/:fingerprint/:index/* - let hd_key_route = warp::path!("hd" / Fingerprint / u32 / ..) + let hd_key_route = warp::path!("hd" / DescriptorChecksum / u32 / ..) .and(query.clone()) - .map(|fingerprint: Fingerprint, index: u32, query: Arc| { - let script_info = query - .get_hd_script_info(fingerprint, index) - .or_err(StatusCode::NOT_FOUND)?; - Ok(script_info.scripthash) - }) + .map( + |desc_check: DescriptorChecksum, index: u32, query: Arc| { + let script_info = query + .get_hd_script_info(desc_check, index) + .or_err(StatusCode::NOT_FOUND)?; + Ok(script_info.scripthash) + }, + ) .and_then(reject_error); let spk_route = address_route diff --git a/src/query.rs b/src/query.rs index 7f3731d..0806651 100644 --- a/src/query.rs +++ b/src/query.rs @@ -5,7 +5,6 @@ use std::time::{Duration, Instant}; use serde::Serialize; use serde_json::Value; -use bitcoin::util::bip32::Fingerprint; use bitcoin::{BlockHash, BlockHeader, Network, OutPoint, Txid}; use bitcoincore_rpc::{json as rpcjson, Client as RpcClient, RpcApi}; @@ -17,7 +16,7 @@ use crate::types::{BlockId, MempoolEntry, ScriptHash, TxStatus}; use crate::util::make_fee_histogram; #[cfg(feature = "track-spends")] -use crate::types::InPoint; +use crate::types::{DescriptorChecksum, InPoint}; const FEE_HISTOGRAM_TTL: Duration = Duration::from_secs(120); const FEE_ESTIMATES_TTL: Duration = Duration::from_secs(120); @@ -415,38 +414,43 @@ impl Query { // HD Wallets // - pub fn get_hd_wallets(&self) -> HashMap { + pub fn get_hd_wallets(&self) -> HashMap { self.indexer.read().unwrap().watcher().wallets().clone() } - pub fn get_hd_wallet(&self, fingerprint: Fingerprint) -> Option { + pub fn get_hd_wallet(&self, desc_check: DescriptorChecksum) -> Option { self.indexer .read() .unwrap() .watcher() - .get(fingerprint) + .get(desc_check) .cloned() } // get the ScriptInfo entry of a derived hd key, without it necessarily being indexed - pub fn get_hd_script_info(&self, fingerprint: Fingerprint, index: u32) -> Option { + pub fn get_hd_script_info( + &self, + desc_check: DescriptorChecksum, + index: u32, + ) -> Option { let indexer = self.indexer.read().unwrap(); - let wallet = indexer.watcher().get(fingerprint)?; - let key = wallet.derive(index); - let address = wallet.to_address(&key); + let wallet = indexer.watcher().get(desc_check.clone()); + let address = wallet.unwrap().derive_address(index, &self.rpc).unwrap(); let scripthash = ScriptHash::from(&address); - let origin = KeyOrigin::Derived(fingerprint, index); + let origin = KeyOrigin::Derived(desc_check.clone(), index); Some(ScriptInfo::new(scripthash, address, origin)) } - pub fn find_hd_gap(&self, fingerprint: Fingerprint) -> Option { + pub fn find_hd_gap(&self, desc_check: DescriptorChecksum) -> Option { let indexer = self.indexer.read().unwrap(); let store = indexer.store(); - let wallet = indexer.watcher().get(fingerprint)?; + let wallet = indexer.watcher().get(desc_check)?; let max_funded_index = wallet.max_funded_index?; // return None if this wallet has no history at all let gap = (0..=max_funded_index) - .map(|derivation_index| ScriptHash::from(&wallet.derive_address(derivation_index))) + .map(|derivation_index| { + ScriptHash::from(&wallet.derive_address(derivation_index, &self.rpc).unwrap()) + }) .fold((0, 0), |(curr_gap, max_gap), scripthash| { if store.has_history(&scripthash) { (0, curr_gap.max(max_gap)) diff --git a/src/types.rs b/src/types.rs index 4f834b3..3f2f2c0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,10 +1,14 @@ use std::cmp::Ordering; +use std::fmt; +use std::str::FromStr; use serde::Serialize; +use crate::error::BwtError; use bitcoin::{Address, BlockHash, Txid}; use bitcoin_hashes::{sha256, Hash}; pub use bitcoincore_rpc::json::ImportMultiRescanSince as RescanSince; +use bitcoincore_rpc::{Client as RpcClient, RpcApi}; use crate::bitcoincore_ext::GetMempoolEntryResult; @@ -180,3 +184,35 @@ impl From for MempoolEntry { } } } + +#[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] +pub struct DescriptorChecksum(pub String); + +impl FromStr for DescriptorChecksum { + type Err = BwtError; + fn from_str(s: &str) -> Result { + Ok(DescriptorChecksum(s.to_string())) + } +} + +#[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] +pub struct Descriptor(pub String); + +impl Descriptor { + pub fn new(descriptor: &str, rpc: &RpcClient) -> Result { + // TODO: what to do about non-ranged descriptors? + let info = rpc.get_descriptor_info(descriptor).unwrap(); + Ok(Self(info.descriptor)) + } + + pub fn checksum(&self) -> DescriptorChecksum { + // This assumes that the descriptor is legit ... + DescriptorChecksum(self.0.split("#").collect::>()[1].to_string()) + } +} + +impl fmt::Display for DescriptorChecksum { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} From 01729c76cb03ca38f8459c5915e23e5db80eed59 Mon Sep 17 00:00:00 2001 From: Justin Moon Date: Thu, 25 Jun 2020 17:40:06 -0500 Subject: [PATCH 2/4] Code review fixes - desc_check -> descr_cs - 'hd'-> 'wallet' in HTTP API - Return `Result` where appropriate - Replaced a few `.unwrap` with `?` - Fix origin mapping and remove KeyOrigin::DerivedHard - HDWallet.from_config accepts descriptors, xpubs, and bare xpubs - Swap CLI -d and -D shorthands --- src/app.rs | 12 +++-- src/config.rs | 8 +-- src/hd.rs | 145 ++++++++++++++++++++++++++++++++------------------ src/http.rs | 73 +++++++++++-------------- src/query.rs | 33 +++++------- src/types.rs | 23 ++++---- 6 files changed, 162 insertions(+), 132 deletions(-) diff --git a/src/app.rs b/src/app.rs index 68be2ee..56d30c1 100644 --- a/src/app.rs +++ b/src/app.rs @@ -36,17 +36,21 @@ impl App { pub fn boot(config: Config) -> Result { debug!("{:?}", config); - let rpc = RpcClient::new(config.bitcoind_url(), config.bitcoind_auth()?)?; + let rpc = Arc::new(RpcClient::new( + config.bitcoind_url(), + config.bitcoind_auth()?, + )?); - let wallets = HDWallet::from_descriptors( + let wallets = HDWallet::from_config( &config.descriptors[..], + &config.xpubs[..], + &config.bare_xpubs[..], config.network, config.gap_limit, config.initial_import_size, - &rpc, + rpc.clone(), )?; - let rpc = Arc::new(rpc); let watcher = HDWatcher::new(wallets); let indexer = Arc::new(RwLock::new(Indexer::new(rpc.clone(), watcher))); diff --git a/src/config.rs b/src/config.rs index 84d6551..c985bef 100644 --- a/src/config.rs +++ b/src/config.rs @@ -56,7 +56,7 @@ pub struct Config { pub bitcoind_wallet: Option, #[structopt( - short = "d", + short = "D", long, help = "Path to bitcoind directory (used for cookie file) [default: ~/.bitcoin]", env, @@ -97,9 +97,9 @@ pub struct Config { pub bitcoind_cookie: Option, #[structopt( - short = "D", - long = "descriptors", - help = "descriptors to track and since when (rescans from genesis by default, use : or : to specify a timestmap, or :none to disable rescan)", + short = "d", + long = "descriptor", + help = "Output script descriptor to track and since when (rescans from genesis by default, use : or : to specify a timestmap, or :none to disable rescan)", parse(try_from_str = parse_descriptor), env, hide_env_values(true), use_delimiter(true), display_order(20) diff --git a/src/hd.rs b/src/hd.rs index 96bcf63..9b3db5d 100644 --- a/src/hd.rs +++ b/src/hd.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::result::Result as StdResult; use std::str::FromStr; +use std::sync::Arc; use serde::Serialize; @@ -11,7 +12,7 @@ use bitcoincore_rpc::json::ImportMultiRequest; use bitcoincore_rpc::{self as rpc, Client as RpcClient, RpcApi}; use crate::error::{Context, Result}; -use crate::types::{Descriptor, DescriptorChecksum, RescanSince, ScriptType}; +use crate::types::{DescrChecksum, Descriptor, RescanSince, ScriptType}; const LABEL_PREFIX: &str = "bwt"; @@ -21,7 +22,7 @@ lazy_static! { #[derive(Debug)] pub struct HDWatcher { - wallets: HashMap, + wallets: HashMap, } impl HDWatcher { @@ -34,18 +35,18 @@ impl HDWatcher { } } - pub fn wallets(&self) -> &HashMap { + pub fn wallets(&self) -> &HashMap { &self.wallets } - pub fn get(&self, desc_check: DescriptorChecksum) -> Option<&HDWallet> { - self.wallets.get(&desc_check) + pub fn get(&self, descr_cs: DescrChecksum) -> Option<&HDWallet> { + self.wallets.get(&descr_cs) } // Mark an address as funded pub fn mark_funded(&mut self, origin: &KeyOrigin) { - if let KeyOrigin::Derived(desc_check, index) = origin { - if let Some(wallet) = self.wallets.get_mut(desc_check) { + if let KeyOrigin::Derived(descr_cs, index) = origin { + if let Some(wallet) = self.wallets.get_mut(descr_cs) { if wallet.max_imported_index.map_or(true, |max| *index > max) { wallet.max_imported_index = Some(*index); } @@ -61,25 +62,25 @@ impl HDWatcher { pub fn check_imports(&mut self, rpc: &RpcClient) -> Result<()> { debug!("checking previous imports"); let labels: Vec = rpc.call("listlabels", &[]).map_err(labels_error)?; - let mut imported_indexes: HashMap = HashMap::new(); + let mut imported_indexes: HashMap = HashMap::new(); for label in labels { - if let Some(KeyOrigin::Derived(desc_check, index)) = KeyOrigin::from_label(&label) { - if self.wallets.contains_key(&desc_check) { + if let Some(KeyOrigin::Derived(descr_cs, index)) = KeyOrigin::from_label(&label) { + if self.wallets.contains_key(&descr_cs) { imported_indexes - .entry(desc_check) + .entry(descr_cs) .and_modify(|current| *current = (*current).max(index)) .or_insert(index); } } } - for (desc_check, max_imported_index) in imported_indexes { + for (descr_cs, max_imported_index) in imported_indexes { trace!( "wallet {} was imported up to index {}", - desc_check, + descr_cs, max_imported_index ); - let wallet = self.wallets.get_mut(&desc_check).unwrap(); + let wallet = self.wallets.get_mut(&descr_cs).unwrap(); wallet.max_imported_index = Some(max_imported_index); // if anything was imported at all, assume we've finished the initial sync. this might @@ -94,7 +95,7 @@ impl HDWatcher { let mut import_reqs = vec![]; let mut pending_updates = vec![]; - for (desc_check, wallet) in self.wallets.iter_mut() { + for (descr_cs, wallet) in self.wallets.iter_mut() { let watch_index = wallet.watch_index(); if wallet.max_imported_index.map_or(true, |i| watch_index > i) { let start_index = wallet @@ -103,21 +104,21 @@ impl HDWatcher { debug!( "importing {} range {}-{} with rescan={}", - desc_check, start_index, watch_index, rescan, + descr_cs, start_index, watch_index, rescan, ); import_reqs.append(&mut wallet.make_imports(start_index, watch_index, rescan)); - pending_updates.push((wallet, desc_check, watch_index)); + pending_updates.push((wallet, descr_cs, watch_index)); } else if !wallet.done_initial_import { debug!( "done initial import for {} up to index {}", - desc_check, + descr_cs, wallet.max_imported_index.unwrap() ); wallet.done_initial_import = true; } else { - trace!("no imports needed for {}", desc_check); + trace!("no imports needed for {}", descr_cs); } } @@ -133,8 +134,8 @@ impl HDWatcher { info!("done importing batch"); } - for (wallet, desc_check, imported_index) in pending_updates { - debug!("imported {} up to index {}", desc_check, imported_index); + for (wallet, descr_cs, imported_index) in pending_updates { + debug!("imported {} up to index {}", descr_cs, imported_index); wallet.max_imported_index = Some(imported_index); } @@ -176,13 +177,16 @@ impl HDWallet { } } - pub fn from_descriptors( + pub fn from_config( descriptors: &[(String, RescanSince)], + xpubs: &[(XyzPubKey, RescanSince)], + bare_xpubs: &[(XyzPubKey, RescanSince)], network: Network, gap_limit: u32, initial_import_size: u32, - rpc: &RpcClient, + rpc: Arc, ) -> Result> { + // Descriptors let mut wallets = vec![]; for (descriptor, rescan) in descriptors { wallets.push( @@ -192,12 +196,64 @@ impl HDWallet { gap_limit, initial_import_size, *rescan, - rpc, + rpc.clone(), ) .with_context(|| format!("invalid descriptor {}", descriptor))? .clone(), ); } + + // Xpubs + for (xyz, rescan) in xpubs { + // Change and receiving output descriptors + for change in 0..2 { + let descriptor = match xyz.script_type { + ScriptType::P2pkh => { + format!("pkh({}/{}/*)", xyz.extended_pubkey.to_string(), change) + } + ScriptType::P2wpkh => { + format!("wpkh({}/{}/*)", xyz.extended_pubkey.to_string(), change) + } + ScriptType::P2shP2wpkh => { + format!("sh(wpkh({}/{}/*))", xyz.extended_pubkey.to_string(), change) + } + }; + wallets.push( + Self::from_descriptor( + descriptor.clone(), + network, + gap_limit, + initial_import_size, + *rescan, + rpc.clone(), + ) + .with_context(|| format!("Invalid xpub-derived descriptor {}", descriptor))? + .clone(), + ); + } + } + + // Bare xpubs + for (xyz, rescan) in bare_xpubs { + let descriptor = match xyz.script_type { + ScriptType::P2pkh => format!("pkh({})", xyz.extended_pubkey.to_string()), + ScriptType::P2wpkh => format!("wpkh({})", xyz.extended_pubkey.to_string()), + ScriptType::P2shP2wpkh => format!("sh(wpkh({}))", xyz.extended_pubkey.to_string()), + }; + wallets.push( + Self::from_descriptor( + descriptor.clone(), + network, + gap_limit, + initial_import_size, + *rescan, + rpc.clone(), + ) + .with_context(|| format!("Invalid bare-xpub-derived descriptor {}", descriptor))? + .clone(), + ); + } + if wallets.is_empty() { warn!("Please provide at least one descriptor to track (via --descriptors)."); bail!("No descriptors provided"); @@ -211,17 +267,9 @@ impl HDWallet { gap_limit: u32, initial_import_size: u32, rescan_policy: RescanSince, - rpc: &RpcClient, + rpc: Arc, ) -> Result { - let descriptor = Descriptor::new(&descriptor, rpc).unwrap(); - // FIXME - //ensure!( - //xpub.matches_network(network), - //"xpub network mismatch, {} is {} and not {}", - //xpub, - //xpub.network, - //network - //); + let descriptor = Descriptor::new(&descriptor, rpc)?; Ok(Self::new( descriptor, network, @@ -257,7 +305,7 @@ impl HDWallet { (start_index..=end_index) .map(|index| { - let label = format!("{}/{}", self.descriptor.checksum(), index); + let label = KeyOrigin::Derived(self.descriptor.checksum(), index).to_label(); (self.descriptor.clone(), index, label, rescan_since) }) .collect() @@ -308,10 +356,8 @@ fn batch_import( #[derive(Debug, Clone, PartialEq)] pub enum KeyOrigin { - Derived(DescriptorChecksum, u32), + Derived(DescrChecksum, u32), Standalone, - // bwt never does hardended derivation itself, but can receive an hardend --bare-xpub - DerivedHard(DescriptorChecksum, u32), } impl_string_serializer!( @@ -319,11 +365,8 @@ impl_string_serializer!( origin, match origin { KeyOrigin::Standalone => "standalone".into(), - KeyOrigin::Derived(parent_desc_check, index) => { - format!("{}/{}", parent_desc_check, index) - } - KeyOrigin::DerivedHard(parent_desc_check, index) => { - format!("{}/{}'", parent_desc_check, index) + KeyOrigin::Derived(parent_descr_cs, index) => { + format!("{}/{}", parent_descr_cs, index) } } ); @@ -331,22 +374,18 @@ impl_string_serializer!( impl KeyOrigin { pub fn to_label(&self) -> String { match self { - KeyOrigin::Derived(parent_desc_check, index) => format!( - "{}/{}/{}", - LABEL_PREFIX, - hex::encode(parent_desc_check.0.as_bytes()), - index - ), + KeyOrigin::Derived(descr_cs, index) => { + format!("{}/{}/{}", LABEL_PREFIX, descr_cs, index) + } KeyOrigin::Standalone => LABEL_PREFIX.into(), - KeyOrigin::DerivedHard(..) => unreachable!(), } } pub fn from_label(s: &str) -> Option { let parts: Vec<&str> = s.splitn(3, '/').collect(); match (parts.get(0), parts.get(1), parts.get(2)) { - (Some(parent), Some(index), None) => Some(KeyOrigin::Derived( - DescriptorChecksum(parent.to_string()), + (Some(&LABEL_PREFIX), Some(parent), Some(index)) => Some(KeyOrigin::Derived( + DescrChecksum(parent.to_string()), index.parse().ok()?, )), (Some(&LABEL_PREFIX), None, None) => Some(KeyOrigin::Standalone), @@ -357,7 +396,7 @@ impl KeyOrigin { pub fn is_standalone(origin: &KeyOrigin) -> bool { match origin { KeyOrigin::Standalone => true, - KeyOrigin::Derived(..) | KeyOrigin::DerivedHard(..) => false, + KeyOrigin::Derived(..) => false, } } } diff --git a/src/http.rs b/src/http.rs index 0d4f0b5..7a5e351 100644 --- a/src/http.rs +++ b/src/http.rs @@ -13,7 +13,7 @@ use bitcoin::{Address, BlockHash, OutPoint, Txid}; use bitcoin_hashes::hex::FromHex; use crate::error::{fmt_error_chain, BwtError, Error, OptionExt}; -use crate::types::{BlockId, DescriptorChecksum, ScriptHash}; +use crate::types::{BlockId, DescrChecksum, ScriptHash}; use crate::{store, IndexChange, Query}; type SyncChanSender = Arc>>; @@ -41,63 +41,56 @@ async fn run( } // GET /hd - let hd_wallets_handler = - warp::get() - .and(warp::path!("hd")) - .and(query.clone()) - .map(|query: Arc| { - let wallets = query.get_hd_wallets(); - reply::json(&wallets) - }); + let hd_wallets_handler = warp::get() + .and(warp::path!("wallet")) + .and(query.clone()) + .map(|query: Arc| { + let wallets = query.get_hd_wallets(); + reply::json(&wallets) + }); // GET /hd/:fingerprint let hd_wallet_handler = warp::get() - .and(warp::path!("hd" / DescriptorChecksum)) + .and(warp::path!("wallet" / DescrChecksum)) .and(query.clone()) - .map(|desc_check: DescriptorChecksum, query: Arc| { - let wallet = query - .get_hd_wallet(desc_check) - .or_err(StatusCode::NOT_FOUND)?; + .map(|descr_cs: DescrChecksum, query: Arc| { + let wallet = query.get_hd_wallet(descr_cs).or_err(StatusCode::NOT_FOUND)?; Ok(reply::json(&wallet)) }) .map(handle_error); // GET /hd/:fingerprint/:index let hd_key_handler = warp::get() - .and(warp::path!("hd" / DescriptorChecksum / u32)) + .and(warp::path!("wallet" / DescrChecksum / u32)) .and(query.clone()) - .map( - |desc_check: DescriptorChecksum, index: u32, query: Arc| { - let script_info = query - .get_hd_script_info(desc_check, index) - .or_err(StatusCode::NOT_FOUND)?; - Ok(reply::json(&script_info)) - }, - ) + .map(|descr_cs: DescrChecksum, index: u32, query: Arc| { + let script_info = query + .get_hd_script_info(descr_cs, index)? + .or_err(StatusCode::NOT_FOUND)?; + Ok(reply::json(&script_info)) + }) .map(handle_error); // GET /hd/:fingerprint/gap let hd_gap_handler = warp::get() - .and(warp::path!("hd" / DescriptorChecksum / "gap")) + .and(warp::path!("wallet" / DescrChecksum / "gap")) .and(query.clone()) - .map(|desc_check: DescriptorChecksum, query: Arc| { - let gap = query - .find_hd_gap(desc_check) - .or_err(StatusCode::NOT_FOUND)?; + .map(|descr_cs: DescrChecksum, query: Arc| { + let gap = query.find_hd_gap(descr_cs)?.or_err(StatusCode::NOT_FOUND)?; Ok(reply::json(&gap)) }) .map(handle_error); // GET /hd/:fingerprint/next let hd_next_handler = warp::get() - .and(warp::path!("hd" / DescriptorChecksum / "next")) + .and(warp::path!("wallet" / DescrChecksum / "next")) .and(query.clone()) - .map(|desc_check: DescriptorChecksum, query: Arc| { + .map(|descr_cs: DescrChecksum, query: Arc| { let wallet = query - .get_hd_wallet(desc_check.clone()) + .get_hd_wallet(descr_cs.clone()) .or_err(StatusCode::NOT_FOUND)?; let next_index = wallet.get_next_index(); - let uri = format!("/hd/{}/{}", desc_check.clone(), next_index); + let uri = format!("/hd/{}/{}", descr_cs, next_index); // issue a 307 redirect to the hdkey resource uri, and also include the derivation // index in the response Ok(reply::with_header( @@ -116,16 +109,14 @@ async fn run( // TODO check address version bytes matches the configured network // GET /hd/:fingerprint/:index/* - let hd_key_route = warp::path!("hd" / DescriptorChecksum / u32 / ..) + let hd_key_route = warp::path!("wallet" / DescrChecksum / u32 / ..) .and(query.clone()) - .map( - |desc_check: DescriptorChecksum, index: u32, query: Arc| { - let script_info = query - .get_hd_script_info(desc_check, index) - .or_err(StatusCode::NOT_FOUND)?; - Ok(script_info.scripthash) - }, - ) + .map(|descr_cs: DescrChecksum, index: u32, query: Arc| { + let script_info = query + .get_hd_script_info(descr_cs, index)? + .or_err(StatusCode::NOT_FOUND)?; + Ok(script_info.scripthash) + }) .and_then(reject_error); let spk_route = address_route diff --git a/src/query.rs b/src/query.rs index 0806651..ca8f386 100644 --- a/src/query.rs +++ b/src/query.rs @@ -16,7 +16,7 @@ use crate::types::{BlockId, MempoolEntry, ScriptHash, TxStatus}; use crate::util::make_fee_histogram; #[cfg(feature = "track-spends")] -use crate::types::{DescriptorChecksum, InPoint}; +use crate::types::{DescrChecksum, InPoint}; const FEE_HISTOGRAM_TTL: Duration = Duration::from_secs(120); const FEE_ESTIMATES_TTL: Duration = Duration::from_secs(120); @@ -414,38 +414,33 @@ impl Query { // HD Wallets // - pub fn get_hd_wallets(&self) -> HashMap { + pub fn get_hd_wallets(&self) -> HashMap { self.indexer.read().unwrap().watcher().wallets().clone() } - pub fn get_hd_wallet(&self, desc_check: DescriptorChecksum) -> Option { - self.indexer - .read() - .unwrap() - .watcher() - .get(desc_check) - .cloned() + pub fn get_hd_wallet(&self, descr_cs: DescrChecksum) -> Option { + self.indexer.read().unwrap().watcher().get(descr_cs).cloned() } // get the ScriptInfo entry of a derived hd key, without it necessarily being indexed pub fn get_hd_script_info( &self, - desc_check: DescriptorChecksum, + descr_cs: DescrChecksum, index: u32, - ) -> Option { + ) -> Result> { let indexer = self.indexer.read().unwrap(); - let wallet = indexer.watcher().get(desc_check.clone()); - let address = wallet.unwrap().derive_address(index, &self.rpc).unwrap(); + let wallet = some_or_ret!(indexer.watcher().get(descr_cs.clone()), Ok(None)); + let address = wallet.derive_address(index, &self.rpc)?; let scripthash = ScriptHash::from(&address); - let origin = KeyOrigin::Derived(desc_check.clone(), index); - Some(ScriptInfo::new(scripthash, address, origin)) + let origin = KeyOrigin::Derived(descr_cs.clone(), index); + Ok(Some(ScriptInfo::new(scripthash, address, origin))) } - pub fn find_hd_gap(&self, desc_check: DescriptorChecksum) -> Option { + pub fn find_hd_gap(&self, descr_cs: DescrChecksum) -> Result> { let indexer = self.indexer.read().unwrap(); let store = indexer.store(); - let wallet = indexer.watcher().get(desc_check)?; - let max_funded_index = wallet.max_funded_index?; // return None if this wallet has no history at all + let wallet = some_or_ret!(indexer.watcher().get(descr_cs), Ok(None)); + let max_funded_index = some_or_ret!(wallet.max_funded_index, Ok(None)); // return None if this wallet has no history at all let gap = (0..=max_funded_index) .map(|derivation_index| { @@ -459,7 +454,7 @@ impl Query { } }) .1; - Some(gap) + Ok(Some(gap)) } } diff --git a/src/types.rs b/src/types.rs index 3f2f2c0..98a15dc 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,6 +1,7 @@ use std::cmp::Ordering; use std::fmt; use std::str::FromStr; +use std::sync::Arc; use serde::Serialize; @@ -186,32 +187,32 @@ impl From for MempoolEntry { } #[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] -pub struct DescriptorChecksum(pub String); +pub struct DescrChecksum(pub String); -impl FromStr for DescriptorChecksum { +impl FromStr for DescrChecksum { type Err = BwtError; - fn from_str(s: &str) -> Result { - Ok(DescriptorChecksum(s.to_string())) + fn from_str(s: &str) -> Result { + Ok(DescrChecksum(s.to_string())) } } +/// A *ranged* output script descriptor #[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] pub struct Descriptor(pub String); impl Descriptor { - pub fn new(descriptor: &str, rpc: &RpcClient) -> Result { - // TODO: what to do about non-ranged descriptors? - let info = rpc.get_descriptor_info(descriptor).unwrap(); + pub fn new(descriptor: &str, rpc: Arc) -> Result { + let info = rpc.get_descriptor_info(descriptor)?; Ok(Self(info.descriptor)) } - pub fn checksum(&self) -> DescriptorChecksum { - // This assumes that the descriptor is legit ... - DescriptorChecksum(self.0.split("#").collect::>()[1].to_string()) + pub fn checksum(&self) -> DescrChecksum { + // This assumes that the descriptor is valid ... + DescrChecksum(self.0.split("#").collect::>()[1].to_string()) } } -impl fmt::Display for DescriptorChecksum { +impl fmt::Display for DescrChecksum { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.0) } From 3a3d0b7f71e5a4a8e471d567425bfbff3a3c6b76 Mon Sep 17 00:00:00 2001 From: Justin Moon Date: Thu, 25 Jun 2020 20:38:21 -0500 Subject: [PATCH 3/4] More code review - Descriptor has "body" and "checksum" attributes - Forgot a /* turning bare xpubs into descriptors - Removed unnecessary clones - Rename "hd" -> "wallet" in a few more places --- src/hd.rs | 35 +++++++++++++++++------------------ src/http.rs | 30 ++++++++++++++++-------------- src/query.rs | 7 ++++++- src/types.rs | 28 +++++++++++++--------------- 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/hd.rs b/src/hd.rs index 9b3db5d..5235ef6 100644 --- a/src/hd.rs +++ b/src/hd.rs @@ -30,7 +30,7 @@ impl HDWatcher { HDWatcher { wallets: wallets .into_iter() - .map(|wallet| (wallet.descriptor.checksum(), wallet)) + .map(|wallet| (wallet.descriptor.checksum.clone(), wallet)) .collect(), } } @@ -198,15 +198,14 @@ impl HDWallet { *rescan, rpc.clone(), ) - .with_context(|| format!("invalid descriptor {}", descriptor))? - .clone(), + .with_context(|| format!("invalid descriptor {}", descriptor))?, ); } // Xpubs for (xyz, rescan) in xpubs { // Change and receiving output descriptors - for change in 0..2 { + for change in 0..=1 { let descriptor = match xyz.script_type { ScriptType::P2pkh => { format!("pkh({}/{}/*)", xyz.extended_pubkey.to_string(), change) @@ -227,8 +226,7 @@ impl HDWallet { *rescan, rpc.clone(), ) - .with_context(|| format!("Invalid xpub-derived descriptor {}", descriptor))? - .clone(), + .with_context(|| format!("Invalid xpub-derived descriptor {}", descriptor))?, ); } } @@ -236,9 +234,11 @@ impl HDWallet { // Bare xpubs for (xyz, rescan) in bare_xpubs { let descriptor = match xyz.script_type { - ScriptType::P2pkh => format!("pkh({})", xyz.extended_pubkey.to_string()), - ScriptType::P2wpkh => format!("wpkh({})", xyz.extended_pubkey.to_string()), - ScriptType::P2shP2wpkh => format!("sh(wpkh({}))", xyz.extended_pubkey.to_string()), + ScriptType::P2pkh => format!("pkh({})/*", xyz.extended_pubkey.to_string()), + ScriptType::P2wpkh => format!("wpkh({}/*)", xyz.extended_pubkey.to_string()), + ScriptType::P2shP2wpkh => { + format!("sh(wpkh({}/*))", xyz.extended_pubkey.to_string()) + } }; wallets.push( Self::from_descriptor( @@ -249,8 +249,7 @@ impl HDWallet { *rescan, rpc.clone(), ) - .with_context(|| format!("Invalid bare-xpub-derived descriptor {}", descriptor))? - .clone(), + .with_context(|| format!("Invalid bare-xpub-derived descriptor {}", descriptor))?, ); } @@ -296,7 +295,7 @@ impl HDWallet { start_index: u32, end_index: u32, rescan: bool, - ) -> Vec<(Descriptor, u32, String, RescanSince)> { + ) -> Vec<(String, u32, String, RescanSince)> { let rescan_since = if rescan { self.rescan_policy } else { @@ -305,14 +304,14 @@ impl HDWallet { (start_index..=end_index) .map(|index| { - let label = KeyOrigin::Derived(self.descriptor.checksum(), index).to_label(); - (self.descriptor.clone(), index, label, rescan_since) + let label = KeyOrigin::Derived(self.descriptor.checksum.clone(), index).to_label(); + (self.descriptor.to_string(), index, label, rescan_since) }) .collect() } pub fn derive_address(&self, index: u32, rpc: &RpcClient) -> Result
{ - let res = rpc.derive_addresses(&self.descriptor.0, Some([index, index]))?; + let res = rpc.derive_addresses(&self.descriptor.to_string(), Some([index, index]))?; Ok(res[0].to_owned()) } @@ -323,7 +322,7 @@ impl HDWallet { } fn batch_import( rpc: &RpcClient, - import_reqs: Vec<(Descriptor, u32, String, RescanSince)>, + import_reqs: Vec<(String, u32, String, RescanSince)>, ) -> Result<()> { let results = rpc.import_multi( &import_reqs @@ -334,7 +333,7 @@ fn batch_import( watchonly: Some(true), timestamp: *rescan_since, range: Some((*index as usize, *index as usize)), - descriptor: Some(&descriptor.0), + descriptor: Some(&descriptor), ..Default::default() }, ) @@ -489,7 +488,7 @@ impl Serialize for HDWallet { S: serde::Serializer, { let mut rgb = serializer.serialize_struct("HDWallet", 3)?; - rgb.serialize_field("descriptor", &self.descriptor.0)?; + rgb.serialize_field("descriptor", &self.descriptor.body)?; rgb.serialize_field("network", &self.network)?; rgb.serialize_field("gap_limit", &self.gap_limit)?; rgb.serialize_field("initial_import_size", &self.initial_import_size)?; diff --git a/src/http.rs b/src/http.rs index 7a5e351..6b3ae2d 100644 --- a/src/http.rs +++ b/src/http.rs @@ -40,7 +40,7 @@ async fn run( ); } - // GET /hd + // GET /wallet let hd_wallets_handler = warp::get() .and(warp::path!("wallet")) .and(query.clone()) @@ -49,17 +49,19 @@ async fn run( reply::json(&wallets) }); - // GET /hd/:fingerprint + // GET /wallet/:fingerprint let hd_wallet_handler = warp::get() .and(warp::path!("wallet" / DescrChecksum)) .and(query.clone()) .map(|descr_cs: DescrChecksum, query: Arc| { - let wallet = query.get_hd_wallet(descr_cs).or_err(StatusCode::NOT_FOUND)?; + let wallet = query + .get_hd_wallet(descr_cs) + .or_err(StatusCode::NOT_FOUND)?; Ok(reply::json(&wallet)) }) .map(handle_error); - // GET /hd/:fingerprint/:index + // GET /wallet/:fingerprint/:index let hd_key_handler = warp::get() .and(warp::path!("wallet" / DescrChecksum / u32)) .and(query.clone()) @@ -71,7 +73,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/gap + // GET /wallet/:fingerprint/gap let hd_gap_handler = warp::get() .and(warp::path!("wallet" / DescrChecksum / "gap")) .and(query.clone()) @@ -81,7 +83,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/next + // GET /wallet/:fingerprint/next let hd_next_handler = warp::get() .and(warp::path!("wallet" / DescrChecksum / "next")) .and(query.clone()) @@ -90,7 +92,7 @@ async fn run( .get_hd_wallet(descr_cs.clone()) .or_err(StatusCode::NOT_FOUND)?; let next_index = wallet.get_next_index(); - let uri = format!("/hd/{}/{}", descr_cs, next_index); + let uri = format!("/wallet/{}/{}", descr_cs, next_index); // issue a 307 redirect to the hdkey resource uri, and also include the derivation // index in the response Ok(reply::with_header( @@ -108,7 +110,7 @@ async fn run( let address_route = warp::path!("address" / Address / ..).map(ScriptHash::from); // TODO check address version bytes matches the configured network - // GET /hd/:fingerprint/:index/* + // GET /wallet/:fingerprint/:index/* let hd_key_route = warp::path!("wallet" / DescrChecksum / u32 / ..) .and(query.clone()) .map(|descr_cs: DescrChecksum, index: u32, query: Arc| { @@ -125,7 +127,7 @@ async fn run( .or(hd_key_route) .unify(); - // GET /hd/:fingerprint/:index + // GET /wallet/:fingerprint/:index // GET /address/:address // GET /scripthash/:scripthash let spk_handler = warp::get() @@ -140,7 +142,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/:index/stats + // GET /wallet/:fingerprint/:index/stats // GET /address/:address/stats // GET /scripthash/:scripthash/stats let spk_stats_handler = warp::get() @@ -155,7 +157,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/:index/utxos + // GET /wallet/:fingerprint/:index/utxos // GET /address/:address/utxos // GET /scripthash/:scripthash/utxos let spk_utxo_handler = warp::get() @@ -170,7 +172,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/:index/txs + // GET /wallet/:fingerprint/:index/txs // GET /address/:address/txs // GET /scripthash/:scripthash/txs let spk_txs_handler = warp::get() @@ -185,7 +187,7 @@ async fn run( }) .map(handle_error); - // GET /hd/:fingerprint/:index/txs/compact + // GET /wallet/:fingerprint/:index/txs/compact // GET /address/:address/txs/compact // GET /scripthash/:scripthash/txs/compact let spk_txs_compact_handler = warp::get() @@ -312,7 +314,7 @@ async fn run( ) .map(handle_error); - // GET /hd/:fingerprint/:index/stream + // GET /wallet/:fingerprint/:index/stream // GET /scripthash/:scripthash/stream // GET /address/:address/stream let spk_sse_handler = warp::get() diff --git a/src/query.rs b/src/query.rs index ca8f386..2ea74bf 100644 --- a/src/query.rs +++ b/src/query.rs @@ -419,7 +419,12 @@ impl Query { } pub fn get_hd_wallet(&self, descr_cs: DescrChecksum) -> Option { - self.indexer.read().unwrap().watcher().get(descr_cs).cloned() + self.indexer + .read() + .unwrap() + .watcher() + .get(descr_cs) + .cloned() } // get the ScriptInfo entry of a derived hd key, without it necessarily being indexed diff --git a/src/types.rs b/src/types.rs index 98a15dc..1d8311e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,5 +1,4 @@ use std::cmp::Ordering; -use std::fmt; use std::str::FromStr; use std::sync::Arc; @@ -186,7 +185,7 @@ impl From for MempoolEntry { } } -#[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] +#[derive(Clone, Eq, PartialEq, Debug, Hash)] pub struct DescrChecksum(pub String); impl FromStr for DescrChecksum { @@ -196,24 +195,23 @@ impl FromStr for DescrChecksum { } } +impl_string_serializer!(DescrChecksum, descr_cs, descr_cs.0); + /// A *ranged* output script descriptor -#[derive(Serialize, Clone, Eq, PartialEq, Debug, Hash)] -pub struct Descriptor(pub String); +#[derive(Clone, Eq, PartialEq, Debug, Hash)] +pub struct Descriptor { + pub body: String, + pub checksum: DescrChecksum, +} impl Descriptor { pub fn new(descriptor: &str, rpc: Arc) -> Result { let info = rpc.get_descriptor_info(descriptor)?; - Ok(Self(info.descriptor)) - } - - pub fn checksum(&self) -> DescrChecksum { - // This assumes that the descriptor is valid ... - DescrChecksum(self.0.split("#").collect::>()[1].to_string()) + Ok(Self { + body: descriptor.to_string(), + checksum: DescrChecksum(info.checksum), + }) } } -impl fmt::Display for DescrChecksum { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } -} +impl_string_serializer!(Descriptor, desc, format!("{}#{}", desc.body, desc.checksum)); From 5877ad03d50e7faf392429808318c280987705ae Mon Sep 17 00:00:00 2001 From: Justin Moon Date: Thu, 25 Jun 2020 21:17:51 -0500 Subject: [PATCH 4/4] HDWallet::from_descriptor -> HDWallet::new --- src/hd.rs | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/src/hd.rs b/src/hd.rs index 5235ef6..593c90e 100644 --- a/src/hd.rs +++ b/src/hd.rs @@ -157,26 +157,6 @@ pub struct HDWallet { } impl HDWallet { - pub fn new( - descriptor: Descriptor, - network: Network, - gap_limit: u32, - initial_import_size: u32, - rescan_policy: RescanSince, - ) -> Self { - Self { - descriptor, - network, - gap_limit, - // setting initial_import_size < gap_limit makes no sense, the user probably meant to increase both - initial_import_size: initial_import_size.max(gap_limit), - rescan_policy, - done_initial_import: false, - max_funded_index: None, - max_imported_index: None, - } - } - pub fn from_config( descriptors: &[(String, RescanSince)], xpubs: &[(XyzPubKey, RescanSince)], @@ -190,7 +170,7 @@ impl HDWallet { let mut wallets = vec![]; for (descriptor, rescan) in descriptors { wallets.push( - Self::from_descriptor( + Self::new( descriptor.clone(), network, gap_limit, @@ -218,7 +198,7 @@ impl HDWallet { } }; wallets.push( - Self::from_descriptor( + Self::new( descriptor.clone(), network, gap_limit, @@ -241,7 +221,7 @@ impl HDWallet { } }; wallets.push( - Self::from_descriptor( + Self::new( descriptor.clone(), network, gap_limit, @@ -260,7 +240,7 @@ impl HDWallet { Ok(wallets) } - pub fn from_descriptor( + pub fn new( descriptor: String, network: Network, gap_limit: u32, @@ -269,13 +249,17 @@ impl HDWallet { rpc: Arc, ) -> Result { let descriptor = Descriptor::new(&descriptor, rpc)?; - Ok(Self::new( + Ok(Self { descriptor, network, gap_limit, - initial_import_size, + // setting initial_import_size < gap_limit makes no sense, the user probably meant to increase both + initial_import_size: initial_import_size.max(gap_limit), rescan_policy, - )) + done_initial_import: false, + max_funded_index: None, + max_imported_index: None, + }) } /// Returns the maximum index that needs to be watched