From 1cece1b468f4a10a971bdee103621f43eb0a41c6 Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Thu, 7 Sep 2023 11:48:37 +0530 Subject: [PATCH 01/38] logging channel_id of connected txids --- lightning/src/ln/channelmanager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 5f4ea2969f9..14b3fd716bb 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -7326,6 +7326,7 @@ where fn handle_channel_ready(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelReady) { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); let _ = handle_error!(self, self.internal_channel_ready(counterparty_node_id, msg), *counterparty_node_id); + log_trace!(self.logger, "connected txids Channel_id {}", msg.channel_id); } fn handle_shutdown(&self, counterparty_node_id: &PublicKey, msg: &msgs::Shutdown) { From b2236a6a393a4b7c0c2786d024286155f29a1925 Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Sat, 9 Sep 2023 16:00:42 +0530 Subject: [PATCH 02/38] logging confirmed txid --- lightning/src/ln/channelmanager.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 14b3fd716bb..d620d03e426 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -6992,6 +6992,12 @@ where let timestamp = self.highest_seen_timestamp.load(Ordering::Acquire); self.do_chain_event(Some(last_best_block_height), |channel| channel.best_block_updated(last_best_block_height, timestamp as u32, self.genesis_hash.clone(), &self.node_signer, &self.default_configuration, &self.logger)); } + // log each confirmed tansaction's Txid. + for tx in txdata.iter() { + let (index, transaction) = tx; + let txid = transaction.txid(); + log_trace!(self.logger, "Transaction id {} confirmed in block {}", txid, block_hash); + } } fn best_block_updated(&self, header: &BlockHeader, height: u32) { @@ -7326,7 +7332,6 @@ where fn handle_channel_ready(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelReady) { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); let _ = handle_error!(self, self.internal_channel_ready(counterparty_node_id, msg), *counterparty_node_id); - log_trace!(self.logger, "connected txids Channel_id {}", msg.channel_id); } fn handle_shutdown(&self, counterparty_node_id: &PublicKey, msg: &msgs::Shutdown) { From 743f548db0bca65e0f53e51fd2d29841a17650e2 Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Sun, 17 Sep 2023 23:36:17 +0530 Subject: [PATCH 03/38] logging in channel monitor --- lightning/src/chain/channelmonitor.rs | 3 +++ lightning/src/ln/channelmanager.rs | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 47f5605edbb..c5575bd9602 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -3390,6 +3390,8 @@ impl ChannelMonitorImpl { let mut balance_spendable_csv = None; log_info!(logger, "Channel {} closed by funding output spend in txid {}.", &self.funding_info.0.to_channel_id(), txid); + // logging only that spends funding output + log_trace!(logger, "Transaction id {} confirmed in block {}", txid , block_hash); self.funding_spend_seen = true; let mut commitment_tx_to_counterparty_output = None; if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 { @@ -3438,6 +3440,7 @@ impl ChannelMonitorImpl { if let Some(new_outputs) = new_outputs_option { watch_outputs.push(new_outputs); } + log_trace!(logger, "Transaction id {} confirmed in block {}", commitment_txid , block_hash); // Since there may be multiple HTLCs for this channel (all spending the // same commitment tx) being claimed by the counterparty within the same // transaction, and `check_spend_counterparty_htlc` already checks all the diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index f1f7a8a58e6..3ea9301007b 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -7009,12 +7009,6 @@ where let timestamp = self.highest_seen_timestamp.load(Ordering::Acquire); self.do_chain_event(Some(last_best_block_height), |channel| channel.best_block_updated(last_best_block_height, timestamp as u32, self.genesis_hash.clone(), &self.node_signer, &self.default_configuration, &self.logger)); } - // log each confirmed tansaction's Txid. - for tx in txdata.iter() { - let (index, transaction) = tx; - let txid = transaction.txid(); - log_trace!(self.logger, "Transaction id {} confirmed in block {}", txid, block_hash); - } } fn best_block_updated(&self, header: &BlockHeader, height: u32) { From 10ea10ea548d6807ce21507dc3fe024d25e8af3a Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Mon, 18 Sep 2023 10:41:38 +0530 Subject: [PATCH 04/38] logging all filtered txids --- lightning/src/chain/channelmonitor.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index c5575bd9602..2a9094aedb4 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -3351,6 +3351,7 @@ impl ChannelMonitorImpl { let mut claimable_outpoints = Vec::new(); 'tx_iter: for tx in &txn_matched { let txid = tx.txid(); + log_trace!(logger, "Transaction id {} confirmed in block {}", txid , block_hash); // If a transaction has already been confirmed, ensure we don't bother processing it duplicatively. if Some(txid) == self.funding_spend_confirmed { log_debug!(logger, "Skipping redundant processing of funding-spend tx {} as it was previously confirmed", txid); @@ -3390,8 +3391,6 @@ impl ChannelMonitorImpl { let mut balance_spendable_csv = None; log_info!(logger, "Channel {} closed by funding output spend in txid {}.", &self.funding_info.0.to_channel_id(), txid); - // logging only that spends funding output - log_trace!(logger, "Transaction id {} confirmed in block {}", txid , block_hash); self.funding_spend_seen = true; let mut commitment_tx_to_counterparty_output = None; if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 { @@ -3440,7 +3439,6 @@ impl ChannelMonitorImpl { if let Some(new_outputs) = new_outputs_option { watch_outputs.push(new_outputs); } - log_trace!(logger, "Transaction id {} confirmed in block {}", commitment_txid , block_hash); // Since there may be multiple HTLCs for this channel (all spending the // same commitment tx) being claimed by the counterparty within the same // transaction, and `check_spend_counterparty_htlc` already checks all the From 3477eeabc77e4ce5157835cf69a13e18afd19c19 Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Wed, 27 Sep 2023 23:03:23 +0530 Subject: [PATCH 05/38] adding txid to filter and logging best_block_update --- lightning/src/chain/chainmonitor.rs | 2 +- lightning/src/chain/channelmonitor.rs | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index fef1e3bf14f..15f6f3f48e1 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -700,7 +700,7 @@ where C::Target: chain::Filter, } } if let Some(ref chain_source) = self.chain_source { - monitor.load_outputs_to_watch(chain_source); + monitor.load_outputs_to_watch(chain_source, &self.logger); } entry.insert(MonitorHolder { monitor, diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 2a9094aedb4..7c747c6432f 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1349,7 +1349,15 @@ impl ChannelMonitor { /// Loads the funding txo and outputs to watch into the given `chain::Filter` by repeatedly /// calling `chain::Filter::register_output` and `chain::Filter::register_tx` until all outputs /// have been registered. - pub fn load_outputs_to_watch(&self, filter: &F) where F::Target: chain::Filter { + pub fn load_outputs_to_watch( + &self, + filter: &F, + logger: &L, + ) + where + F::Target: chain::Filter, + L::Target: Logger, + { let lock = self.inner.lock().unwrap(); filter.register_tx(&lock.get_funding_txo().0.txid, &lock.get_funding_txo().1); for (txid, outputs) in lock.get_outputs_to_watch().iter() { @@ -1360,6 +1368,7 @@ impl ChannelMonitor { outpoint: OutPoint { txid: *txid, index: *index as u16 }, script_pubkey: script_pubkey.clone(), }); + log_trace!(logger, "Adding txid {} to the filter", txid); } } } @@ -3312,9 +3321,11 @@ impl ChannelMonitorImpl { if height > self.best_block.height() { self.best_block = BestBlock::new(block_hash, height); + log_trace!(logger, "New best_block of height {} has been found and updated", height); self.block_confirmed(height, block_hash, vec![], vec![], vec![], &broadcaster, &fee_estimator, &logger) } else if block_hash != self.best_block.block_hash() { self.best_block = BestBlock::new(block_hash, height); + log_trace!(logger, "New best_block of block hash {} has been found and updated", block_hash); self.onchain_events_awaiting_threshold_conf.retain(|ref entry| entry.height <= height); self.onchain_tx_handler.block_disconnected(height + 1, broadcaster, fee_estimator, logger); Vec::new() @@ -3351,7 +3362,7 @@ impl ChannelMonitorImpl { let mut claimable_outpoints = Vec::new(); 'tx_iter: for tx in &txn_matched { let txid = tx.txid(); - log_trace!(logger, "Transaction id {} confirmed in block {}", txid , block_hash); + log_trace!(logger, "Transaction id {} confirmed in block {}", txid , block_hash); // If a transaction has already been confirmed, ensure we don't bother processing it duplicatively. if Some(txid) == self.funding_spend_confirmed { log_debug!(logger, "Skipping redundant processing of funding-spend tx {} as it was previously confirmed", txid); From 8f1e8b6f5afe3e123b0d080715a01f2d55cf7ac7 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Mon, 19 Dec 2022 02:02:22 -0600 Subject: [PATCH 06/38] Handle if funding output is in a coinbase transaction --- lightning/src/ln/channel.rs | 33 ++++++++++++++++++++++++------ lightning/src/ln/channelmanager.rs | 12 ++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index da1364021ce..022767e6f61 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -594,6 +594,9 @@ pub(crate) const DISCONNECT_PEER_AWAITING_RESPONSE_TICKS: usize = 2; /// exceeding this age limit will be force-closed and purged from memory. pub(crate) const UNFUNDED_CHANNEL_AGE_LIMIT_TICKS: usize = 60; +/// Number of blocks needed for an output from a coinbase transaction to be spendable. +pub(crate) const COINBASE_MATURITY: u32 = 100; + struct PendingChannelMonitorUpdate { update: ChannelMonitorUpdate, } @@ -4734,12 +4737,14 @@ impl Channel where return Err(ClosureReason::ProcessingError { err: err_reason.to_owned() }); } else { if self.context.is_outbound() { - for input in tx.input.iter() { - if input.witness.is_empty() { - // We generated a malleable funding transaction, implying we've - // just exposed ourselves to funds loss to our counterparty. - #[cfg(not(fuzzing))] - panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!"); + if !tx.is_coin_base() { + for input in tx.input.iter() { + if input.witness.is_empty() { + // We generated a malleable funding transaction, implying we've + // just exposed ourselves to funds loss to our counterparty. + #[cfg(not(fuzzing))] + panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!"); + } } } } @@ -4750,6 +4755,13 @@ impl Channel where Err(_) => panic!("Block was bogus - either height was > 16 million, had > 16 million transactions, or had > 65k outputs"), } } + // If this is a coinbase transaction and not a 0-conf channel + // we should update our min_depth to 100 to handle coinbase maturity + if tx.is_coin_base() && + self.context.minimum_depth.unwrap_or(0) > 0 && + self.context.minimum_depth.unwrap_or(0) < COINBASE_MATURITY { + self.context.minimum_depth = Some(COINBASE_MATURITY); + } } // If we allow 1-conf funding, we may need to check for channel_ready here and // send it immediately instead of waiting for a best_block_updated call (which @@ -5821,6 +5833,15 @@ impl OutboundV1Channel where SP::Target: SignerProvider { self.context.channel_state = ChannelState::FundingCreated as u32; self.context.channel_id = funding_txo.to_channel_id(); + + // If the funding transaction is a coinbase transaction, we need to set the minimum depth to 100. + // We can skip this if it is a zero-conf channel. + if funding_transaction.is_coin_base() && + self.context.minimum_depth.unwrap_or(0) > 0 && + self.context.minimum_depth.unwrap_or(0) < COINBASE_MATURITY { + self.context.minimum_depth = Some(COINBASE_MATURITY); + } + self.context.funding_transaction = Some(funding_transaction); let channel = Channel { diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 213a2882fbc..017625954cc 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -3561,11 +3561,13 @@ where pub fn funding_transaction_generated(&self, temporary_channel_id: &ChannelId, counterparty_node_id: &PublicKey, funding_transaction: Transaction) -> Result<(), APIError> { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); - for inp in funding_transaction.input.iter() { - if inp.witness.is_empty() { - return Err(APIError::APIMisuseError { - err: "Funding transaction must be fully signed and spend Segwit outputs".to_owned() - }); + if !funding_transaction.is_coin_base() { + for inp in funding_transaction.input.iter() { + if inp.witness.is_empty() { + return Err(APIError::APIMisuseError { + err: "Funding transaction must be fully signed and spend Segwit outputs".to_owned() + }); + } } } { From 3b30a800e9cd3d243424c1af1832b67f2ddd6238 Mon Sep 17 00:00:00 2001 From: benthecarman Date: Wed, 19 Apr 2023 03:53:54 -0500 Subject: [PATCH 07/38] Support creating coinbase funding transactions in tests --- lightning/src/ln/functional_test_utils.rs | 32 ++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index b8c2e085999..947375b1248 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -46,7 +46,7 @@ use alloc::rc::Rc; use crate::sync::{Arc, Mutex, LockTestExt, RwLock}; use core::mem; use core::iter::repeat; -use bitcoin::{PackedLockTime, TxMerkleNode}; +use bitcoin::{PackedLockTime, TxIn, TxMerkleNode}; pub const CHAN_CONFIRM_DEPTH: u32 = 10; @@ -1005,7 +1005,23 @@ macro_rules! reload_node { }; } -pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, expected_counterparty_node_id: &PublicKey, expected_chan_value: u64, expected_user_chan_id: u128) -> (ChannelId, Transaction, OutPoint) { +pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, + expected_counterparty_node_id: &PublicKey, expected_chan_value: u64, expected_user_chan_id: u128) + -> (ChannelId, Transaction, OutPoint) +{ + internal_create_funding_transaction(node, expected_counterparty_node_id, expected_chan_value, expected_user_chan_id, false) +} + +pub fn create_coinbase_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, + expected_counterparty_node_id: &PublicKey, expected_chan_value: u64, expected_user_chan_id: u128) + -> (ChannelId, Transaction, OutPoint) +{ + internal_create_funding_transaction(node, expected_counterparty_node_id, expected_chan_value, expected_user_chan_id, true) +} + +fn internal_create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, + expected_counterparty_node_id: &PublicKey, expected_chan_value: u64, expected_user_chan_id: u128, + coinbase: bool) -> (ChannelId, Transaction, OutPoint) { let chan_id = *node.network_chan_count.borrow(); let events = node.node.get_and_clear_pending_events(); @@ -1016,7 +1032,16 @@ pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, expected_ assert_eq!(*channel_value_satoshis, expected_chan_value); assert_eq!(user_channel_id, expected_user_chan_id); - let tx = Transaction { version: chan_id as i32, lock_time: PackedLockTime::ZERO, input: Vec::new(), output: vec![TxOut { + let input = if coinbase { + vec![TxIn { + previous_output: bitcoin::OutPoint::null(), + ..Default::default() + }] + } else { + Vec::new() + }; + + let tx = Transaction { version: chan_id as i32, lock_time: PackedLockTime::ZERO, input, output: vec![TxOut { value: *channel_value_satoshis, script_pubkey: output_script.clone(), }]}; let funding_outpoint = OutPoint { txid: tx.txid(), index: 0 }; @@ -1025,6 +1050,7 @@ pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, expected_ _ => panic!("Unexpected event"), } } + pub fn sign_funding_transaction<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a, 'b, 'c>, channel_value: u64, expected_temporary_channel_id: ChannelId) -> Transaction { let (temporary_channel_id, tx, funding_output) = create_funding_transaction(node_a, &node_b.node.get_our_node_id(), channel_value, 42); assert_eq!(temporary_channel_id, expected_temporary_channel_id); From 87ac1bfce8d11beb4befa7a3328efd1a600836bd Mon Sep 17 00:00:00 2001 From: Duncan Dean Date: Fri, 1 Sep 2023 11:13:57 +0200 Subject: [PATCH 08/38] Add test for coinbase funding transactions --- lightning/src/ln/functional_tests.rs | 62 +++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index d3401e5f068..a5a55bc2da3 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -20,7 +20,7 @@ use crate::chain::transaction::OutPoint; use crate::sign::{ChannelSigner, EcdsaChannelSigner, EntropySource, SignerProvider}; use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose, ClosureReason, HTLCDestination, PaymentFailureReason}; use crate::ln::{ChannelId, PaymentPreimage, PaymentSecret, PaymentHash}; -use crate::ln::channel::{commitment_tx_base_weight, COMMITMENT_TX_WEIGHT_PER_HTLC, CONCURRENT_INBOUND_HTLC_FEE_BUFFER, FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE, MIN_AFFORDABLE_HTLC_COUNT, get_holder_selected_channel_reserve_satoshis, OutboundV1Channel, InboundV1Channel}; +use crate::ln::channel::{commitment_tx_base_weight, COMMITMENT_TX_WEIGHT_PER_HTLC, CONCURRENT_INBOUND_HTLC_FEE_BUFFER, FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE, MIN_AFFORDABLE_HTLC_COUNT, get_holder_selected_channel_reserve_satoshis, OutboundV1Channel, InboundV1Channel, COINBASE_MATURITY}; use crate::ln::channelmanager::{self, PaymentId, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, BREAKDOWN_TIMEOUT, ENABLE_GOSSIP_TICKS, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA}; use crate::ln::channel::{DISCONNECT_PEER_AWAITING_RESPONSE_TICKS, ChannelError}; use crate::ln::{chan_utils, onion_utils}; @@ -9133,6 +9133,66 @@ fn test_invalid_funding_tx() { mine_transaction(&nodes[1], &spend_tx); } +#[test] +fn test_coinbase_funding_tx() { + // Miners are able to fund channels directly from coinbase transactions, however + // by consensus rules, outputs of a coinbase transaction are encumbered by a 100 + // block maturity timelock. To ensure that a (non-0conf) channel like this is enforceable + // on-chain, the minimum depth is updated to 100 blocks for coinbase funding transactions. + // + // Note that 0conf channels with coinbase funding transactions are unaffected and are + // immediately operational after opening. + let chanmon_cfgs = create_chanmon_cfgs(2); + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 42, None).unwrap(); + let open_channel = get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()); + + nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), &open_channel); + let accept_channel = get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()); + + nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), &accept_channel); + + // Create the coinbase funding transaction. + let (temporary_channel_id, tx, _) = create_coinbase_funding_transaction(&nodes[0], &nodes[1].node.get_our_node_id(), 100000, 42); + + nodes[0].node.funding_transaction_generated(&temporary_channel_id, &nodes[1].node.get_our_node_id(), tx.clone()).unwrap(); + check_added_monitors!(nodes[0], 0); + let funding_created = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()); + + nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created); + check_added_monitors!(nodes[1], 1); + expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id()); + + let funding_signed = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()); + + nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed); + check_added_monitors!(nodes[0], 1); + + expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id()); + assert!(nodes[0].node.get_and_clear_pending_events().is_empty()); + + // Starting at height 0, we "confirm" the coinbase at height 1. + confirm_transaction_at(&nodes[0], &tx, 1); + // We connect 98 more blocks to have 99 confirmations for the coinbase transaction. + connect_blocks(&nodes[0], COINBASE_MATURITY - 2); + // Check that we have no pending message events (we have not queued a `channel_ready` yet). + assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty()); + // Now connect one more block which results in 100 confirmations of the coinbase transaction. + connect_blocks(&nodes[0], 1); + // There should now be a `channel_ready` which can be handled. + let _ = &nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &get_event_msg!(&nodes[0], MessageSendEvent::SendChannelReady, nodes[1].node.get_our_node_id())); + + confirm_transaction_at(&nodes[1], &tx, 1); + connect_blocks(&nodes[1], COINBASE_MATURITY - 2); + assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty()); + connect_blocks(&nodes[1], 1); + expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id()); + create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]); +} + fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_timelock: bool) { // In the first version of the chain::Confirm interface, after a refactor was made to not // broadcast CSV-locked transactions until their CSV lock is up, we wouldn't reliably broadcast From fe0a7fccb0e1f9a2b1f93af7645f1ce393535dfe Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 31 Aug 2023 16:04:33 +0200 Subject: [PATCH 09/38] Add constructor to `RouteParameters` --- lightning/src/routing/router.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 79d54e22d67..a279568cda8 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -435,6 +435,13 @@ pub struct RouteParameters { pub final_value_msat: u64, } +impl RouteParameters { + /// Constructs [`RouteParameters`] from the given [`PaymentParameters`] and a payment amount. + pub fn from_payment_params_and_value(payment_params: PaymentParameters, final_value_msat: u64) -> Self { + Self { payment_params, final_value_msat } + } +} + impl Writeable for RouteParameters { fn write(&self, writer: &mut W) -> Result<(), io::Error> { write_tlv_fields!(writer, { From 203a847d28d7c202bc8ddfe058b8caaaa4997263 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 31 Aug 2023 12:25:38 +0200 Subject: [PATCH 10/38] Have `get_route` take `RouteParameters` --- fuzz/src/full_stack.rs | 12 +- fuzz/src/router.rs | 15 +- lightning-invoice/src/utils.rs | 12 +- lightning/src/ln/channelmanager.rs | 33 +- lightning/src/ln/functional_test_utils.rs | 27 +- lightning/src/ln/functional_tests.rs | 19 +- lightning/src/ln/onion_route_tests.rs | 7 +- lightning/src/ln/outbound_payment.rs | 15 +- lightning/src/ln/payment_tests.rs | 159 ++---- lightning/src/ln/shutdown_tests.rs | 13 +- lightning/src/routing/router.rs | 579 +++++++++++++++------- 11 files changed, 519 insertions(+), 372 deletions(-) diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index a11e5c81368..d7167146cac 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -527,10 +527,8 @@ pub fn do_test(data: &[u8], logger: &Arc) { 4 => { let final_value_msat = slice_to_be24(get_slice!(3)) as u64; let payment_params = PaymentParameters::from_node_id(get_pubkey!(), 42); - let params = RouteParameters { - payment_params, - final_value_msat, - }; + let params = RouteParameters::from_payment_params_and_value( + payment_params, final_value_msat); let mut payment_hash = PaymentHash([0; 32]); payment_hash.0[0..8].copy_from_slice(&be64_to_array(payments_sent)); let mut sha = Sha256::engine(); @@ -548,10 +546,8 @@ pub fn do_test(data: &[u8], logger: &Arc) { 15 => { let final_value_msat = slice_to_be24(get_slice!(3)) as u64; let payment_params = PaymentParameters::from_node_id(get_pubkey!(), 42); - let params = RouteParameters { - payment_params, - final_value_msat, - }; + let params = RouteParameters::from_payment_params_and_value( + payment_params, final_value_msat); let mut payment_hash = PaymentHash([0; 32]); payment_hash.0[0..8].copy_from_slice(&be64_to_array(payments_sent)); let mut sha = Sha256::engine(); diff --git a/fuzz/src/router.rs b/fuzz/src/router.rs index 830f6f4e201..b7d45bf729f 100644 --- a/fuzz/src/router.rs +++ b/fuzz/src/router.rs @@ -326,11 +326,10 @@ pub fn do_test(data: &[u8], out: Out) { let mut last_hops = Vec::new(); last_hops!(last_hops); find_routes!(first_hops, node_pks.iter(), |final_amt, final_delta, target: &PublicKey| { - RouteParameters { - payment_params: PaymentParameters::from_node_id(*target, final_delta) + RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(*target, final_delta) .with_route_hints(last_hops.clone()).unwrap(), - final_value_msat: final_amt, - } + final_amt) }); }, x => { @@ -366,11 +365,9 @@ pub fn do_test(data: &[u8], out: Out) { let mut features = Bolt12InvoiceFeatures::empty(); features.set_basic_mpp_optional(); find_routes!(first_hops, vec![dummy_pk].iter(), |final_amt, _, _| { - RouteParameters { - payment_params: PaymentParameters::blinded(last_hops.clone()) - .with_bolt12_features(features.clone()).unwrap(), - final_value_msat: final_amt, - } + RouteParameters::from_payment_params_and_value(PaymentParameters::blinded(last_hops.clone()) + .with_bolt12_features(features.clone()).unwrap(), + final_amt) }); } } diff --git a/lightning-invoice/src/utils.rs b/lightning-invoice/src/utils.rs index 744c2654cd8..a512b2de05d 100644 --- a/lightning-invoice/src/utils.rs +++ b/lightning-invoice/src/utils.rs @@ -869,10 +869,8 @@ mod test { invoice.min_final_cltv_expiry_delta() as u32) .with_bolt11_features(invoice.features().unwrap().clone()).unwrap() .with_route_hints(invoice.route_hints()).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: invoice.amount_milli_satoshis().unwrap(), - }; + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, invoice.amount_milli_satoshis().unwrap()); let payment_event = { let mut payment_hash = PaymentHash([0; 32]); payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]); @@ -1326,10 +1324,8 @@ mod test { invoice.min_final_cltv_expiry_delta() as u32) .with_bolt11_features(invoice.features().unwrap().clone()).unwrap() .with_route_hints(invoice.route_hints()).unwrap(); - let params = RouteParameters { - payment_params, - final_value_msat: invoice.amount_milli_satoshis().unwrap(), - }; + let params = RouteParameters::from_payment_params_and_value( + payment_params, invoice.amount_milli_satoshis().unwrap()); let (payment_event, fwd_idx) = { let mut payment_hash = PaymentHash([0; 32]); payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]); diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 017625954cc..cff9837fceb 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -9655,10 +9655,9 @@ mod tests { let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &expected_route, 100_000); // Next, attempt a keysend payment and make sure it fails. - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(expected_route.last().unwrap().node.get_our_node_id(), TEST_FINAL_CLTV, false), - final_value_msat: 100_000, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(expected_route.last().unwrap().node.get_our_node_id(), + TEST_FINAL_CLTV, false), 100_000); let route = find_route( &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph, None, nodes[0].logger, &scorer, &(), &random_seed_bytes @@ -9746,10 +9745,10 @@ mod tests { pass_along_path(&nodes[0], &path, 100_000, payment_hash, None, event, true, Some(payment_preimage)); // Next, attempt a keysend payment and make sure it fails. - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(expected_route.last().unwrap().node.get_our_node_id(), TEST_FINAL_CLTV, false), - final_value_msat: 100_000, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(expected_route.last().unwrap().node.get_our_node_id(), TEST_FINAL_CLTV, false), + 100_000 + ); let route = find_route( &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph, None, nodes[0].logger, &scorer, &(), &random_seed_bytes @@ -9795,10 +9794,8 @@ mod tests { let payee_pubkey = nodes[1].node.get_our_node_id(); let _chan = create_chan_between_nodes(&nodes[0], &nodes[1]); - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, false), - final_value_msat: 10_000, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(payee_pubkey, 40, false), 10_000); let network_graph = nodes[0].network_graph.clone(); let first_hops = nodes[0].node.list_usable_channels(); let scorer = test_utils::TestScorer::new(); @@ -9842,10 +9839,8 @@ mod tests { let payee_pubkey = nodes[1].node.get_our_node_id(); let _chan = create_chan_between_nodes(&nodes[0], &nodes[1]); - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, false), - final_value_msat: 10_000, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(payee_pubkey, 40, false), 10_000); let network_graph = nodes[0].network_graph.clone(); let first_hops = nodes[0].node.list_usable_channels(); let scorer = test_utils::TestScorer::new(); @@ -10742,9 +10737,9 @@ pub mod bench { let payment_secret = $node_b.create_inbound_payment_for_hash(payment_hash, None, 7200, None).unwrap(); $node_a.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), - PaymentId(payment_hash.0), RouteParameters { - payment_params, final_value_msat: 10_000, - }, Retry::Attempts(0)).unwrap(); + PaymentId(payment_hash.0), + RouteParameters::from_payment_params_and_value(payment_params, 10_000), + Retry::Attempts(0)).unwrap(); let payment_event = SendEvent::from_event($node_a.get_and_clear_pending_msg_events().pop().unwrap()); $node_b.handle_update_add_htlc(&$node_a.get_our_node_id(), &payment_event.msgs[0]); $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &payment_event.commitment_msg); diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index 947375b1248..03ec96df740 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -19,7 +19,7 @@ use crate::events::bump_transaction::{BumpTransactionEventHandler, Wallet, Walle use crate::ln::{ChannelId, PaymentPreimage, PaymentHash, PaymentSecret}; use crate::ln::channelmanager::{self, AChannelManager, ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, PaymentId, MIN_CLTV_EXPIRY_DELTA}; use crate::routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate}; -use crate::routing::router::{self, PaymentParameters, Route}; +use crate::routing::router::{self, PaymentParameters, Route, RouteParameters}; use crate::ln::features::InitFeatures; use crate::ln::msgs; use crate::ln::msgs::{ChannelMessageHandler,RoutingMessageHandler}; @@ -1864,14 +1864,14 @@ macro_rules! get_payment_preimage_hash { } /// Gets a route from the given sender to the node described in `payment_params`. -pub fn get_route(send_node: &Node, payment_params: &PaymentParameters, recv_value: u64) -> Result { +pub fn get_route(send_node: &Node, route_params: &RouteParameters) -> Result { let scorer = TestScorer::new(); let keys_manager = TestKeysInterface::new(&[0u8; 32], bitcoin::network::constants::Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); router::get_route( - &send_node.node.get_our_node_id(), payment_params, &send_node.network_graph.read_only(), + &send_node.node.get_our_node_id(), route_params, &send_node.network_graph.read_only(), Some(&send_node.node.list_usable_channels().iter().collect::>()), - recv_value, send_node.logger, &scorer, &(), &random_seed_bytes + send_node.logger, &scorer, &(), &random_seed_bytes ) } @@ -1880,9 +1880,10 @@ pub fn get_route(send_node: &Node, payment_params: &PaymentParameters, recv_valu /// Don't use this, use the identically-named function instead. #[macro_export] macro_rules! get_route { - ($send_node: expr, $payment_params: expr, $recv_value: expr) => { - $crate::ln::functional_test_utils::get_route(&$send_node, &$payment_params, $recv_value) - } + ($send_node: expr, $payment_params: expr, $recv_value: expr) => {{ + let route_params = $crate::routing::router::RouteParameters::from_payment_params_and_value($payment_params, $recv_value); + $crate::ln::functional_test_utils::get_route(&$send_node, &route_params) + }} } #[cfg(test)] @@ -1894,9 +1895,10 @@ macro_rules! get_route_and_payment_hash { $crate::get_route_and_payment_hash!($send_node, $recv_node, payment_params, $recv_value) }}; ($send_node: expr, $recv_node: expr, $payment_params: expr, $recv_value: expr) => {{ + let route_params = $crate::routing::router::RouteParameters::from_payment_params_and_value($payment_params, $recv_value); let (payment_preimage, payment_hash, payment_secret) = $crate::ln::functional_test_utils::get_payment_preimage_hash(&$recv_node, Some($recv_value), None); - let route = $crate::ln::functional_test_utils::get_route(&$send_node, &$payment_params, $recv_value); + let route = $crate::ln::functional_test_utils::get_route(&$send_node, &route_params); (route.unwrap(), payment_hash, payment_preimage, payment_secret) }} } @@ -2490,7 +2492,8 @@ pub const TEST_FINAL_CLTV: u32 = 70; pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) { let payment_params = PaymentParameters::from_node_id(expected_route.last().unwrap().node.get_our_node_id(), TEST_FINAL_CLTV) .with_bolt11_features(expected_route.last().unwrap().node.invoice_features()).unwrap(); - let route = get_route(origin_node, &payment_params, recv_value).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value); + let route = get_route(origin_node, &route_params).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), expected_route.len()); for (node, hop) in expected_route.iter().zip(route.paths[0].hops.iter()) { @@ -2504,14 +2507,14 @@ pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) { let payment_params = PaymentParameters::from_node_id(expected_route.last().unwrap().node.get_our_node_id(), TEST_FINAL_CLTV) .with_bolt11_features(expected_route.last().unwrap().node.invoice_features()).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value); let network_graph = origin_node.network_graph.read_only(); let scorer = test_utils::TestScorer::new(); let seed = [0u8; 32]; let keys_manager = test_utils::TestKeysInterface::new(&seed, Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = router::get_route( - &origin_node.node.get_our_node_id(), &payment_params, &network_graph, - None, recv_value, origin_node.logger, &scorer, &(), &random_seed_bytes).unwrap(); + let route = router::get_route(&origin_node.node.get_our_node_id(), &route_params, &network_graph, + None, origin_node.logger, &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), expected_route.len()); for (node, hop) in expected_route.iter().zip(route.paths[0].hops.iter()) { diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index a5a55bc2da3..849aa042737 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -26,7 +26,7 @@ use crate::ln::channel::{DISCONNECT_PEER_AWAITING_RESPONSE_TICKS, ChannelError}; use crate::ln::{chan_utils, onion_utils}; use crate::ln::chan_utils::{OFFERED_HTLC_SCRIPT_WEIGHT, htlc_success_tx_weight, htlc_timeout_tx_weight, HTLCOutputInCommitment}; use crate::routing::gossip::{NetworkGraph, NetworkUpdate}; -use crate::routing::router::{Path, PaymentParameters, Route, RouteHop, get_route}; +use crate::routing::router::{Path, PaymentParameters, Route, RouteHop, get_route, RouteParameters}; use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, NodeFeatures}; use crate::ln::msgs; use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction}; @@ -7137,8 +7137,11 @@ fn test_check_htlc_underpaying() { let scorer = test_utils::TestScorer::new(); let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes(); - let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV).with_bolt11_features(nodes[1].node.invoice_features()).unwrap(); - let route = get_route(&nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(), None, 10_000, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); + let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), + TEST_FINAL_CLTV).with_bolt11_features(nodes[1].node.invoice_features()).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 10_000); + let route = get_route(&nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph.read_only(), + None, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); let (_, our_payment_hash, _) = get_payment_preimage_hash!(nodes[0]); let our_payment_secret = nodes[1].node.create_inbound_payment_for_hash(our_payment_hash, Some(100_000), 7200, None).unwrap(); nodes[0].node.send_payment_with_route(&route, our_payment_hash, @@ -7394,12 +7397,14 @@ fn test_bump_penalty_txn_on_revoked_htlcs() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), 50).with_bolt11_features(nodes[1].node.invoice_features()).unwrap(); let scorer = test_utils::TestScorer::new(); let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes(); - let route = get_route(&nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(), None, - 3_000_000, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 3_000_000); + let route = get_route(&nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph.read_only(), None, + nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 3_000_000).0; let payment_params = PaymentParameters::from_node_id(nodes[0].node.get_our_node_id(), 50).with_bolt11_features(nodes[0].node.invoice_features()).unwrap(); - let route = get_route(&nodes[1].node.get_our_node_id(), &payment_params, &nodes[1].network_graph.read_only(), None, - 3_000_000, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 3_000_000); + let route = get_route(&nodes[1].node.get_our_node_id(), &route_params, &nodes[1].network_graph.read_only(), None, + nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); send_along_route(&nodes[1], route, &[&nodes[0]], 3_000_000); let revoked_local_txn = get_local_commitment_txn!(nodes[1], chan.2); diff --git a/lightning/src/ln/onion_route_tests.rs b/lightning/src/ln/onion_route_tests.rs index d326c1aab74..469517b02ba 100644 --- a/lightning/src/ln/onion_route_tests.rs +++ b/lightning/src/ln/onion_route_tests.rs @@ -19,7 +19,7 @@ use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS; use crate::ln::channelmanager::{HTLCForwardInfo, FailureCode, CLTV_FAR_FAR_AWAY, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId, RecipientOnionFields}; use crate::ln::onion_utils; use crate::routing::gossip::{NetworkUpdate, RoutingFees}; -use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop}; +use crate::routing::router::{get_route, PaymentParameters, Route, RouteParameters, RouteHint, RouteHintHop}; use crate::ln::features::{InitFeatures, Bolt11InvoiceFeatures}; use crate::ln::msgs; use crate::ln::msgs::{ChannelMessageHandler, ChannelUpdate}; @@ -1048,10 +1048,11 @@ macro_rules! get_phantom_route { ])]).unwrap(); let scorer = test_utils::TestScorer::new(); let network_graph = $nodes[0].network_graph.read_only(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, $amt); (get_route( - &$nodes[0].node.get_our_node_id(), &payment_params, &network_graph, + &$nodes[0].node.get_our_node_id(), &route_params, &network_graph, Some(&$nodes[0].node.list_usable_channels().iter().collect::>()), - $amt, $nodes[0].logger, &scorer, &(), &[0u8; 32] + $nodes[0].logger, &scorer, &(), &[0u8; 32] ).unwrap(), phantom_route_hint.phantom_scid) } }} diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index f60bf565efa..38ae89aec6b 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -1565,10 +1565,7 @@ mod tests { PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()), 0 ).with_expiry_time(past_expiry_time); - let expired_route_params = RouteParameters { - payment_params, - final_value_msat: 0, - }; + let expired_route_params = RouteParameters::from_payment_params_and_value(payment_params, 0); let pending_events = Mutex::new(VecDeque::new()); if on_retry { outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), @@ -1609,10 +1606,7 @@ mod tests { let payment_params = PaymentParameters::from_node_id( PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()), 0); - let route_params = RouteParameters { - payment_params, - final_value_msat: 0, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 0); router.expect_find_route(route_params.clone(), Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })); @@ -1652,10 +1646,7 @@ mod tests { let sender_pk = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()); let receiver_pk = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()); let payment_params = PaymentParameters::from_node_id(sender_pk, 0); - let route_params = RouteParameters { - payment_params: payment_params.clone(), - final_value_msat: 0, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params.clone(), 0); let failed_scid = 42; let route = Route { paths: vec![Path { hops: vec![RouteHop { diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index f0c3fa92d37..f8000423b56 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -94,10 +94,7 @@ fn mpp_retry() { // Initiate the MPP payment. let payment_id = PaymentId(payment_hash.0); - let mut route_params = RouteParameters { - payment_params: route.payment_params.clone().unwrap(), - final_value_msat: amt_msat, - }; + let mut route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), @@ -257,10 +254,8 @@ fn do_test_keysend_payments(public_node: bool, with_retry: bool) { } let payer_pubkey = nodes[0].node.get_our_node_id(); let payee_pubkey = nodes[1].node.get_our_node_id(); - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, false), - final_value_msat: 10000, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(payee_pubkey, 40, false), 10000); let network_graph = nodes[0].network_graph.clone(); let channels = nodes[0].node.list_usable_channels(); @@ -319,10 +314,8 @@ fn test_mpp_keysend() { let payer_pubkey = nodes[0].node.get_our_node_id(); let payee_pubkey = nodes[3].node.get_our_node_id(); let recv_value = 15_000_000; - let route_params = RouteParameters { - payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, true), - final_value_msat: recv_value, - }; + let route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::for_keysend(payee_pubkey, 40, true), recv_value); let scorer = test_utils::TestScorer::new(); let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes(); let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger, @@ -531,10 +524,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) { let amt_msat = 1_000_000; let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat); let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000); - let route_params = RouteParameters { - payment_params: route.payment_params.clone().unwrap(), - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); check_added_monitors!(nodes[0], 1); @@ -1111,10 +1101,11 @@ fn get_ldk_payment_preimage() { let scorer = test_utils::TestScorer::new(); let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = get_route( - &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(), - Some(&nodes[0].node.list_usable_channels().iter().collect::>()), - amt_msat, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); + let route = get_route( &nodes[0].node.get_our_node_id(), &route_params, + &nodes[0].network_graph.read_only(), + Some(&nodes[0].node.list_usable_channels().iter().collect::>()), nodes[0].logger, + &scorer, &(), &random_seed_bytes).unwrap(); nodes[0].node.send_payment_with_route(&route, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap(); check_added_monitors!(nodes[0], 1); @@ -1224,7 +1215,7 @@ fn failed_probe_yields_event() { let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42); - let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000); + let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 9_998_000); let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap(); @@ -1273,7 +1264,7 @@ fn onchain_failed_probe_yields_event() { let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42); // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain. - let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000); + let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 1_000); let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap(); // node[0] -- update_add_htlcs -> node[1] @@ -1661,15 +1652,10 @@ fn do_test_intercepted_payment(test: InterceptTest) { }]) ]).unwrap() .with_bolt11_features(nodes[2].node.invoice_features()).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; - let route = get_route( - &nodes[0].node.get_our_node_id(), &route_params.payment_params, - &nodes[0].network_graph.read_only(), None, route_params.final_value_msat, - nodes[0].logger, &scorer, &(), &random_seed_bytes, - ).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat,); + let route = get_route( &nodes[0].node.get_our_node_id(), &route_params, + &nodes[0].network_graph.read_only(), None, nodes[0].logger, &scorer, &(), + &random_seed_bytes,).unwrap(); let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap(); nodes[0].node.send_payment_with_route(&route, payment_hash, @@ -1850,10 +1836,7 @@ fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) { let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV) .with_route_hints(route_hints).unwrap() .with_bolt11_features(nodes[2].node.invoice_features()).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap(); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap(); @@ -1980,10 +1963,7 @@ fn do_automatic_retries(test: AutoRetry) { let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat); macro_rules! pass_failed_attempt_with_retry_along_path { @@ -2199,10 +2179,7 @@ fn auto_retry_partial_failure() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the // second (for the initial send path2 over chan_2) fails. @@ -2273,14 +2250,14 @@ fn auto_retry_partial_failure() { nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route)); let mut payment_params = route_params.payment_params.clone(); payment_params.previously_failed_channels.push(chan_2_id); - nodes[0].router.expect_find_route(RouteParameters { - payment_params, final_value_msat: amt_msat / 2, - }, Ok(retry_1_route)); + nodes[0].router.expect_find_route( + RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 2), + Ok(retry_1_route)); let mut payment_params = route_params.payment_params.clone(); payment_params.previously_failed_channels.push(chan_3_id); - nodes[0].router.expect_find_route(RouteParameters { - payment_params, final_value_msat: amt_msat / 4, - }, Ok(retry_2_route)); + nodes[0].router.expect_find_route( + RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 4), + Ok(retry_2_route)); // Send a payment that will partially fail on send, then partially fail on retry, then succeed. nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), @@ -2416,10 +2393,7 @@ fn auto_retry_zero_attempts_send_error() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), @@ -2456,10 +2430,7 @@ fn fails_paying_after_rejected_by_payee() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); @@ -2503,10 +2474,8 @@ fn retry_multi_path_single_failed_payment() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params: payment_params.clone(), - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), amt_msat); let chans = nodes[0].node.list_usable_channels(); let mut route = Route { @@ -2536,12 +2505,11 @@ fn retry_multi_path_single_failed_payment() { route.paths[1].hops[0].fee_msat = 50_000_000; let mut pay_params = route.payment_params.clone().unwrap(); pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap()); - nodes[0].router.expect_find_route(RouteParameters { - payment_params: pay_params, - // Note that the second request here requests the amount we originally failed to send, - // not the amount remaining on the full payment, which should be changed. - final_value_msat: 100_000_001, - }, Ok(route.clone())); + nodes[0].router.expect_find_route( + // Note that the second request here requests the amount we originally failed to send, + // not the amount remaining on the full payment, which should be changed. + RouteParameters::from_payment_params_and_value(pay_params, 100_000_001), + Ok(route.clone())); { let scorer = chanmon_cfgs[0].scorer.read().unwrap(); @@ -2596,10 +2564,7 @@ fn immediate_retry_on_failure() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let chans = nodes[0].node.list_usable_channels(); let mut route = Route { @@ -2623,9 +2588,9 @@ fn immediate_retry_on_failure() { route.paths[1].hops[0].fee_msat = 50_000_001; let mut pay_params = route_params.payment_params.clone(); pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap()); - nodes[0].router.expect_find_route(RouteParameters { - payment_params: pay_params, final_value_msat: amt_msat, - }, Ok(route.clone())); + nodes[0].router.expect_find_route( + RouteParameters::from_payment_params_and_value(pay_params, amt_msat), + Ok(route.clone())); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); @@ -2684,10 +2649,7 @@ fn no_extra_retries_on_back_to_back_fail() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let mut route = Route { paths: vec![ @@ -2730,10 +2692,9 @@ fn no_extra_retries_on_back_to_back_fail() { // On retry, we'll only return one path route.paths.remove(1); route.paths[0].hops[1].fee_msat = amt_msat; - nodes[0].router.expect_find_route(RouteParameters { - payment_params: second_payment_params, - final_value_msat: amt_msat, - }, Ok(route.clone())); + nodes[0].router.expect_find_route( + RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat), + Ok(route.clone())); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); @@ -2886,10 +2847,7 @@ fn test_simple_partial_retry() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let mut route = Route { paths: vec![ @@ -2931,10 +2889,9 @@ fn test_simple_partial_retry() { second_payment_params.previously_failed_channels = vec![chan_2_scid]; // On retry, we'll only be asked for one path (or 100k sats) route.paths.remove(0); - nodes[0].router.expect_find_route(RouteParameters { - payment_params: second_payment_params, - final_value_msat: amt_msat / 2, - }, Ok(route.clone())); + nodes[0].router.expect_find_route( + RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat / 2), + Ok(route.clone())); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); @@ -3052,10 +3009,7 @@ fn test_threaded_payment_retries() { let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV) .with_expiry_time(payment_expiry_secs as u64) .with_bolt11_features(invoice_features).unwrap(); - let mut route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); let mut route = Route { paths: vec![ @@ -3303,11 +3257,10 @@ fn do_claim_from_closed_chan(fail_payment: bool) { create_announced_chan_between_nodes(&nodes, 2, 3); let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]); - let mut route_params = RouteParameters { - payment_params: PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV) + let mut route_params = RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV) .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(), - final_value_msat: 10_000_000, - }; + 10_000_000); let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params, None, nodes[0].node.compute_inflight_htlcs()).unwrap(); // Make sure the route is ordered as the B->D path before C->D @@ -3517,10 +3470,7 @@ fn test_retry_custom_tlvs() { // Initiate the payment let payment_id = PaymentId(payment_hash.0); - let mut route_params = RouteParameters { - payment_params: route.payment_params.clone().unwrap(), - final_value_msat: amt_msat, - }; + let mut route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); let custom_tlvs = vec![((1 << 16) + 1, vec![0x42u8; 16])]; let onion_fields = RecipientOnionFields::secret_only(payment_secret); @@ -3772,10 +3722,7 @@ fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) { let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV) .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(); - let mut route_params = RouteParameters { - payment_params, - final_value_msat: amt_msat, - }; + let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat); // Send the MPP payment, delivering the updated commitment state to nodes[1]. nodes[0].node.send_payment(payment_hash, RecipientOnionFields { diff --git a/lightning/src/ln/shutdown_tests.rs b/lightning/src/ln/shutdown_tests.rs index f975e184129..38db58c5549 100644 --- a/lightning/src/ln/shutdown_tests.rs +++ b/lightning/src/ln/shutdown_tests.rs @@ -13,7 +13,7 @@ use crate::sign::{EntropySource, SignerProvider}; use crate::chain::transaction::OutPoint; use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason}; use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId, RecipientOnionFields, ChannelShutdownState, ChannelDetails}; -use crate::routing::router::{PaymentParameters, get_route}; +use crate::routing::router::{PaymentParameters, get_route, RouteParameters}; use crate::ln::msgs; use crate::ln::msgs::{ChannelMessageHandler, ErrorAction}; use crate::ln::script::ShutdownScript; @@ -313,9 +313,14 @@ fn updates_shutdown_wait() { let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[0]); let payment_params_1 = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV).with_bolt11_features(nodes[1].node.invoice_features()).unwrap(); - let route_1 = get_route(&nodes[0].node.get_our_node_id(), &payment_params_1, &nodes[0].network_graph.read_only(), None, 100000, &logger, &scorer, &(), &random_seed_bytes).unwrap(); - let payment_params_2 = PaymentParameters::from_node_id(nodes[0].node.get_our_node_id(), TEST_FINAL_CLTV).with_bolt11_features(nodes[0].node.invoice_features()).unwrap(); - let route_2 = get_route(&nodes[1].node.get_our_node_id(), &payment_params_2, &nodes[1].network_graph.read_only(), None, 100000, &logger, &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params_1, 100_000); + let route_1 = get_route(&nodes[0].node.get_our_node_id(), &route_params, + &nodes[0].network_graph.read_only(), None, &logger, &scorer, &(), &random_seed_bytes).unwrap(); + let payment_params_2 = PaymentParameters::from_node_id(nodes[0].node.get_our_node_id(), + TEST_FINAL_CLTV).with_bolt11_features(nodes[0].node.invoice_features()).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params_2, 100_000); + let route_2 = get_route(&nodes[1].node.get_our_node_id(), &route_params, + &nodes[1].network_graph.read_only(), None, &logger, &scorer, &(), &random_seed_bytes).unwrap(); unwrap_send_err!(nodes[0].node.send_payment_with_route(&route_1, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0) ), true, APIError::ChannelUnavailable {..}, {}); diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index a279568cda8..799b6407981 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -1407,19 +1407,21 @@ pub fn find_route( ) -> Result where L::Target: Logger, GL::Target: Logger { let graph_lock = network_graph.read_only(); - let mut route = get_route(our_node_pubkey, &route_params.payment_params, &graph_lock, first_hops, - route_params.final_value_msat, logger, scorer, score_params, - random_seed_bytes)?; + let mut route = get_route(our_node_pubkey, &route_params, &graph_lock, first_hops, logger, + scorer, score_params, random_seed_bytes)?; add_random_cltv_offset(&mut route, &route_params.payment_params, &graph_lock, random_seed_bytes); Ok(route) } pub(crate) fn get_route( - our_node_pubkey: &PublicKey, payment_params: &PaymentParameters, network_graph: &ReadOnlyNetworkGraph, - first_hops: Option<&[&ChannelDetails]>, final_value_msat: u64, logger: L, scorer: &S, score_params: &S::ScoreParams, + our_node_pubkey: &PublicKey, route_params: &RouteParameters, network_graph: &ReadOnlyNetworkGraph, + first_hops: Option<&[&ChannelDetails]>, logger: L, scorer: &S, score_params: &S::ScoreParams, _random_seed_bytes: &[u8; 32] ) -> Result where L::Target: Logger { + + let payment_params = &route_params.payment_params; + let final_value_msat = route_params.final_value_msat; // If we're routing to a blinded recipient, we won't have their node id. Therefore, keep the // unblinded payee id as an option. We also need a non-optional "payee id" for path construction, // so use a dummy id for this in the blinded case. @@ -2592,17 +2594,15 @@ pub fn build_route_from_hops( ) -> Result where L::Target: Logger, GL::Target: Logger { let graph_lock = network_graph.read_only(); - let mut route = build_route_from_hops_internal( - our_node_pubkey, hops, &route_params.payment_params, &graph_lock, - route_params.final_value_msat, logger, random_seed_bytes)?; + let mut route = build_route_from_hops_internal(our_node_pubkey, hops, &route_params, + &graph_lock, logger, random_seed_bytes)?; add_random_cltv_offset(&mut route, &route_params.payment_params, &graph_lock, random_seed_bytes); Ok(route) } fn build_route_from_hops_internal( - our_node_pubkey: &PublicKey, hops: &[PublicKey], payment_params: &PaymentParameters, - network_graph: &ReadOnlyNetworkGraph, final_value_msat: u64, logger: L, - random_seed_bytes: &[u8; 32] + our_node_pubkey: &PublicKey, hops: &[PublicKey], route_params: &RouteParameters, + network_graph: &ReadOnlyNetworkGraph, logger: L, random_seed_bytes: &[u8; 32], ) -> Result where L::Target: Logger { struct HopScorer { @@ -2649,8 +2649,7 @@ fn build_route_from_hops_internal( let scorer = HopScorer { our_node_id, hop_ids }; - get_route(our_node_pubkey, payment_params, network_graph, None, final_value_msat, - logger, &scorer, &(), random_seed_bytes) + get_route(our_node_pubkey, route_params, network_graph, None, logger, &scorer, &(), random_seed_bytes) } #[cfg(test)] @@ -2660,7 +2659,7 @@ mod tests { use crate::routing::utxo::UtxoResult; use crate::routing::router::{get_route, build_route_from_hops_internal, add_random_cltv_offset, default_node_features, BlindedTail, InFlightHtlcs, Path, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RoutingFees, - DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA, MAX_PATH_LENGTH_ESTIMATE}; + DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA, MAX_PATH_LENGTH_ESTIMATE, RouteParameters}; use crate::routing::scoring::{ChannelUsage, FixedPenaltyScorer, ScoreLookUp, ProbabilisticScorer, ProbabilisticScoringFeeParameters, ProbabilisticScoringDecayParameters}; use crate::routing::test_utils::{add_channel, add_or_update_node, build_graph, build_line_graph, id_to_feature_flags, get_nodes, update_channel}; use crate::chain::transaction::OutPoint; @@ -2743,11 +2742,17 @@ mod tests { // Simple route to 2 via 1 - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 0, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Cannot send a payment of 0 msat"); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 0); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), + &random_seed_bytes) { + assert_eq!(err, "Cannot send a payment of 0 msat"); } else { panic!(); } - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -2778,12 +2783,15 @@ mod tests { let our_chans = vec![get_channel_details(Some(2), our_id, InitFeatures::from_le_bytes(vec![0b11]), 100000)]; - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = - get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "First hop cannot have our_node_pubkey as a destination."); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), + Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "First hop cannot have our_node_pubkey as a destination."); } else { panic!(); } - - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); } @@ -2891,8 +2899,12 @@ mod tests { }); // Not possible to send 199_999_999, because the minimum on channel=2 is 200_000_000. - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 199_999_999, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a path to the given destination"); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 199_999_999); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), + &random_seed_bytes) { + assert_eq!(err, "Failed to find a path to the given destination"); } else { panic!(); } // Lift the restriction on the first hop. @@ -2910,7 +2922,8 @@ mod tests { }); // A payment above the minimum should pass - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 199_999_999, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); } @@ -2992,7 +3005,10 @@ mod tests { excess_data: Vec::new() }); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 60_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 60_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); // Overpay fees to hit htlc_minimum_msat. let overpaid_fees = route.paths[0].hops[0].fee_msat + route.paths[1].hops[0].fee_msat; // TODO: this could be better balanced to overpay 10k and not 15k. @@ -3037,14 +3053,17 @@ mod tests { excess_data: Vec::new() }); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 60_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); // Fine to overpay for htlc_minimum_msat if it allows us to save fee. assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops[0].short_channel_id, 12); let fees = route.paths[0].hops[0].fee_msat; assert_eq!(fees, 5_000); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 50_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 50_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); // Not fine to overpay for htlc_minimum_msat if it requires paying more than fee on // the other channel. assert_eq!(route.paths.len(), 1); @@ -3089,13 +3108,19 @@ mod tests { }); // If all the channels require some features we don't understand, route should fail - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a path to the given destination"); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), + &random_seed_bytes) { + assert_eq!(err, "Failed to find a path to the given destination"); } else { panic!(); } // If we specify a channel to node7, that overrides our local channel view and that gets used - let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), + InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; + let route = get_route(&our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]); @@ -3130,13 +3155,19 @@ mod tests { add_or_update_node(&gossip_sync, &secp_ctx, &privkeys[7], unknown_features.clone(), 1); // If all nodes require some features we don't understand, route should fail - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a path to the given destination"); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), + &random_seed_bytes) { + assert_eq!(err, "Failed to find a path to the given destination"); } else { panic!(); } // If we specify a channel to node7, that overrides our local channel view and that gets used - let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), + InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; + let route = get_route(&our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]); @@ -3168,7 +3199,9 @@ mod tests { // Route to 1 via 2 and 3 because our channel to 1 is disabled let payment_params = PaymentParameters::from_node_id(nodes[0], 42); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 3); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3194,8 +3227,12 @@ mod tests { // If we specify a channel to node7, that overrides our local channel view and that gets used let payment_params = PaymentParameters::from_node_id(nodes[2], 42); - let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), + InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; + let route = get_route(&our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]); @@ -3317,14 +3354,21 @@ mod tests { let mut invalid_last_hops = last_hops_multi_private_channels(&nodes); invalid_last_hops.push(invalid_last_hop); { - let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(invalid_last_hops).unwrap(); - if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Route hint cannot have the payee as the source."); + let payment_params = PaymentParameters::from_node_id(nodes[6], 42) + .with_route_hints(invalid_last_hops).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, + &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), + &random_seed_bytes) { + assert_eq!(err, "Route hint cannot have the payee as the source."); } else { panic!(); } } - let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops_multi_private_channels(&nodes)).unwrap(); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let payment_params = PaymentParameters::from_node_id(nodes[6], 42) + .with_route_hints(last_hops_multi_private_channels(&nodes)).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 5); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3399,8 +3443,9 @@ mod tests { let random_seed_bytes = keys_manager.get_secure_random_bytes(); // Test handling of an empty RouteHint passed in Invoice. - - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 5); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3506,7 +3551,9 @@ mod tests { excess_data: Vec::new() }); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 4); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3578,7 +3625,9 @@ mod tests { excess_data: Vec::new() }); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &[42u8; 32]).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &[42u8; 32]).unwrap(); assert_eq!(route.paths[0].hops.len(), 4); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3660,7 +3709,9 @@ mod tests { // This test shows that public routes can be present in the invoice // which would be handled in the same manner. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 5); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3712,8 +3763,12 @@ mod tests { // Simple test with outbound channel to 4 to test that last_hops and first_hops connect let our_chans = vec![get_channel_details(Some(42), nodes[3].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)]; let mut last_hops = last_hops(&nodes); - let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops.clone()).unwrap(); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let payment_params = PaymentParameters::from_node_id(nodes[6], 42) + .with_route_hints(last_hops.clone()).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 2); assert_eq!(route.paths[0].hops[0].pubkey, nodes[3]); @@ -3733,8 +3788,12 @@ mod tests { last_hops[0].0[0].fees.base_msat = 1000; // Revert to via 6 as the fee on 8 goes up - let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops).unwrap(); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let payment_params = PaymentParameters::from_node_id(nodes[6], 42) + .with_route_hints(last_hops).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 4); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3768,7 +3827,9 @@ mod tests { assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &Vec::::new()); // We can't learn any flags from invoices, sadly // ...but still use 8 for larger payments as 6 has a variable feerate - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 2000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 2000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths[0].hops.len(), 5); assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]); @@ -3833,8 +3894,10 @@ mod tests { let random_seed_bytes = keys_manager.get_secure_random_bytes(); let logger = ln_test_utils::TestLogger::new(); let network_graph = NetworkGraph::new(Network::Testnet, &logger); - let route = get_route(&source_node_id, &payment_params, &network_graph.read_only(), - Some(&our_chans.iter().collect::>()), route_val, &logger, &scorer, &(), &random_seed_bytes); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, route_val); + let route = get_route(&source_node_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), &logger, &scorer, &(), + &random_seed_bytes); route } @@ -3955,15 +4018,21 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 250_000_001); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 250_000_001, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route an exact amount we have should be fine. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 250_000_000, Arc::clone(&logger), &scorer, &(),&random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 250_000_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(),&random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let path = route.paths.last().unwrap(); assert_eq!(path.hops.len(), 2); @@ -3991,15 +4060,23 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 200_000_001); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 200_000_001, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, + &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route an exact amount we have should be fine. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::>()), 200_000_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 200_000_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), + Some(&our_chans.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let path = route.paths.last().unwrap(); assert_eq!(path.hops.len(), 2); @@ -4038,15 +4115,21 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 15_001); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 15_001, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route an exact amount we have should be fine. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 15_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 15_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let path = route.paths.last().unwrap(); assert_eq!(path.hops.len(), 2); @@ -4109,15 +4192,21 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 15_001); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 15_001, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route an exact amount we have should be fine. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 15_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 15_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let path = route.paths.last().unwrap(); assert_eq!(path.hops.len(), 2); @@ -4141,15 +4230,21 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 10_001); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 10_001, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route an exact amount we have should be fine. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 10_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 10_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let path = route.paths.last().unwrap(); assert_eq!(path.hops.len(), 2); @@ -4253,15 +4348,21 @@ mod tests { }); { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 60_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 60_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route 49 sats (just a bit below the capacity). - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 49_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 49_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -4274,7 +4375,10 @@ mod tests { { // Attempt to route an exact amount is also fine - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 50_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 50_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -4322,7 +4426,10 @@ mod tests { }); { - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 50_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 50_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -4487,8 +4594,10 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 300_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 300_000, + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } @@ -4497,8 +4606,10 @@ mod tests { { // Attempt to route while setting max_path_count to 0 results in a failure. let zero_payment_params = payment_params.clone().with_max_path_count(0); + let route_params = RouteParameters::from_payment_params_and_value( + zero_payment_params, 100); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &zero_payment_params, &network_graph.read_only(), None, 100, + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { assert_eq!(err, "Can't find a route with no paths allowed."); } else { panic!(); } @@ -4509,8 +4620,10 @@ mod tests { // This is the case because the minimal_value_contribution_msat would require each path // to account for 1/3 of the total value, which is violated by 2 out of 3 paths. let fail_payment_params = payment_params.clone().with_max_path_count(3); + let route_params = RouteParameters::from_payment_params_and_value( + fail_payment_params, 250_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &fail_payment_params, &network_graph.read_only(), None, 250_000, + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } @@ -4519,8 +4632,10 @@ mod tests { { // Now, attempt to route 250 sats (just a bit below the capacity). // Our algorithm should provide us with these 3 paths. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, - 250_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 250_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 3); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -4537,8 +4652,10 @@ mod tests { { // Attempt to route an exact amount is also fine - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, - 290_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 290_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 3); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -4572,7 +4689,8 @@ mod tests { let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); let config = UserConfig::default(); - let payment_params = PaymentParameters::from_node_id(nodes[3], 42).with_bolt11_features(channelmanager::provided_invoice_features(&config)).unwrap(); + let payment_params = PaymentParameters::from_node_id(nodes[3], 42) + .with_bolt11_features(channelmanager::provided_invoice_features(&config)).unwrap(); // We need a route consisting of 3 paths: // From our node to node3 via {node0, node2}, {node7, node2, node4} and {node7, node2}. @@ -4707,16 +4825,22 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 350_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 350_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route 300 sats (exact amount we can route). // Our algorithm should provide us with these 3 paths, 100 sats each. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 300_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 300_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 3); let mut total_amount_paid_msat = 0; @@ -4877,7 +5001,10 @@ mod tests { { // Now, attempt to route 180 sats. // Our algorithm should provide us with these 2 paths. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 180_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 180_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); let mut total_value_transferred_msat = 0; @@ -5047,15 +5174,20 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 210_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 210_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route 200 sats (exact amount we can route). - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 200_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 200_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); let mut total_amount_paid_msat = 0; @@ -5155,7 +5287,10 @@ mod tests { // Get a route for 100 sats and check that we found the MPP route no problem and didn't // overpay at all. - let mut route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100_000); + let mut route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); route.paths.sort_by_key(|path| path.hops[0].short_channel_id); // Paths are manually ordered ordered by SCID, so: @@ -5273,16 +5408,22 @@ mod tests { { // Attempt to route more than available results in a failure. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 150_000); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 150_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a sufficient route to the given destination"); + &our_id, &route_params, &network_graph.read_only(), None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + assert_eq!(err, "Failed to find a sufficient route to the given destination"); } else { panic!(); } } { // Now, attempt to route 125 sats (just a bit below the capacity of 3 channels). // Our algorithm should provide us with these 3 paths. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 125_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 125_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 3); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -5295,7 +5436,10 @@ mod tests { { // Attempt to route without the last small cheap channel - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 90_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 90_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); let mut total_amount_paid_msat = 0; for path in &route.paths { @@ -5434,7 +5578,10 @@ mod tests { { // Now ensure the route flows simply over nodes 1 and 4 to 6. - let route = get_route(&our_id, &payment_params, &network.read_only(), None, 10_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 10_000); + let route = get_route(&our_id, &route_params, &network.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 3); @@ -5505,7 +5652,10 @@ mod tests { { // Now, attempt to route 90 sats, which is exactly 90 sats at the last hop, plus the // 200% fee charged channel 13 in the 1-to-2 direction. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 90_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 90_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 2); @@ -5571,7 +5721,10 @@ mod tests { // Now, attempt to route 90 sats, hitting the htlc_minimum on channel 4, but // overshooting the htlc_maximum on channel 2. Thus, we should pick the (absurdly // expensive) channels 12-13 path. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 90_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 90_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 2); @@ -5610,10 +5763,12 @@ mod tests { let random_seed_bytes = keys_manager.get_secure_random_bytes(); { - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&[ + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), Some(&[ &get_channel_details(Some(3), nodes[0], channelmanager::provided_init_features(&config), 200_000), &get_channel_details(Some(2), nodes[0], channelmanager::provided_init_features(&config), 10_000), - ]), 100_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + ]), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 1); @@ -5622,10 +5777,12 @@ mod tests { assert_eq!(route.paths[0].hops[0].fee_msat, 100_000); } { - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&[ + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), Some(&[ &get_channel_details(Some(3), nodes[0], channelmanager::provided_init_features(&config), 50_000), &get_channel_details(Some(2), nodes[0], channelmanager::provided_init_features(&config), 50_000), - ]), 100_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + ]), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert_eq!(route.paths[0].hops.len(), 1); assert_eq!(route.paths[1].hops.len(), 1); @@ -5648,7 +5805,9 @@ mod tests { // If we have several options above the 3xpayment value threshold, we should pick the // smallest of them, avoiding further fragmenting our available outbound balance to // this node. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&[ + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), Some(&[ &get_channel_details(Some(2), nodes[0], channelmanager::provided_init_features(&config), 50_000), &get_channel_details(Some(3), nodes[0], channelmanager::provided_init_features(&config), 50_000), &get_channel_details(Some(5), nodes[0], channelmanager::provided_init_features(&config), 50_000), @@ -5657,7 +5816,7 @@ mod tests { &get_channel_details(Some(8), nodes[0], channelmanager::provided_init_features(&config), 50_000), &get_channel_details(Some(9), nodes[0], channelmanager::provided_init_features(&config), 50_000), &get_channel_details(Some(4), nodes[0], channelmanager::provided_init_features(&config), 1_000_000), - ]), 100_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + ]), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 1); @@ -5677,10 +5836,10 @@ mod tests { let scorer = ln_test_utils::TestScorer::new(); let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes - ).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + let route = get_route( &our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert_eq!(route.get_total_fees(), 100); @@ -5690,10 +5849,10 @@ mod tests { // Applying a 100 msat penalty to each hop results in taking channels 7 and 10 to nodes[6] // from nodes[2] rather than channel 6, 11, and 8, even though the longer path is cheaper. let scorer = FixedPenaltyScorer::with_penalty(100); - let route = get_route( - &our_id, &payment_params, &network_graph.read_only(), None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes - ).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100); + let route = get_route( &our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert_eq!(route.get_total_fees(), 300); @@ -5743,10 +5902,10 @@ mod tests { let scorer = ln_test_utils::TestScorer::new(); let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = get_route( - &our_id, &payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes - ).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100); + let route = get_route( &our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert_eq!(route.get_total_fees(), 100); @@ -5755,10 +5914,8 @@ mod tests { // A different path to nodes[6] exists if channel 6 cannot be routed over. let scorer = BadChannelScorer { short_channel_id: 6 }; - let route = get_route( - &our_id, &payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes - ).unwrap(); + let route = get_route( &our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert_eq!(route.get_total_fees(), 300); @@ -5767,14 +5924,12 @@ mod tests { // A path to nodes[6] does not exist if nodes[2] cannot be routed through. let scorer = BadNodeScorer { node_id: NodeId::from_pubkey(&nodes[2]) }; - match get_route( - &our_id, &payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes - ) { - Err(LightningError { err, .. } ) => { - assert_eq!(err, "Failed to find a path to the given destination"); - }, - Ok(_) => panic!("Expected error"), + match get_route( &our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes) { + Err(LightningError { err, .. } ) => { + assert_eq!(err, "Failed to find a path to the given destination"); + }, + Ok(_) => panic!("Expected error"), } } @@ -5863,7 +6018,10 @@ mod tests { .with_max_total_cltv_expiry_delta(feasible_max_total_cltv_delta); let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = get_route(&our_id, &feasible_payment_params, &network_graph, None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + feasible_payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert_ne!(path.len(), 0); @@ -5871,7 +6029,10 @@ mod tests { let fail_max_total_cltv_delta = 23; let fail_payment_params = PaymentParameters::from_node_id(nodes[6], 0).with_route_hints(last_hops(&nodes)).unwrap() .with_max_total_cltv_expiry_delta(fail_max_total_cltv_delta); - match get_route(&our_id, &fail_payment_params, &network_graph, None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) + let route_params = RouteParameters::from_payment_params_and_value( + fail_payment_params, 100); + match get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, + &(), &random_seed_bytes) { Err(LightningError { err, .. } ) => { assert_eq!(err, "Failed to find a path to the given destination"); @@ -5896,9 +6057,16 @@ mod tests { // We should be able to find a route initially, and then after we fail a few random // channels eventually we won't be able to any longer. - assert!(get_route(&our_id, &payment_params, &network_graph, None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).is_ok()); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + assert!(get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).is_ok()); loop { - if let Ok(route) = get_route(&our_id, &payment_params, &network_graph, None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + if let Ok(route) = get_route(&our_id, &route_params, &network_graph, None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes) + { for chan in route.paths[0].hops.iter() { assert!(!payment_params.previously_failed_channels.contains(&chan.short_channel_id)); } @@ -5921,15 +6089,19 @@ mod tests { // First check we can actually create a long route on this graph. let feasible_payment_params = PaymentParameters::from_node_id(nodes[18], 0); - let route = get_route(&our_id, &feasible_payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + feasible_payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::>(); assert!(path.len() == MAX_PATH_LENGTH_ESTIMATE.into()); // But we can't create a path surpassing the MAX_PATH_LENGTH_ESTIMATE limit. let fail_payment_params = PaymentParameters::from_node_id(nodes[19], 0); - match get_route(&our_id, &fail_payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes) + let route_params = RouteParameters::from_payment_params_and_value( + fail_payment_params, 100); + match get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, + &(), &random_seed_bytes) { Err(LightningError { err, .. } ) => { assert_eq!(err, "Failed to find a path to the given destination"); @@ -5948,7 +6120,10 @@ mod tests { let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops(&nodes)).unwrap(); let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); let cltv_expiry_deltas_before = route.paths[0].hops.iter().map(|h| h.cltv_expiry_delta).collect::>(); @@ -5982,8 +6157,10 @@ mod tests { let keys_manager = ln_test_utils::TestKeysInterface::new(&[4u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); - let mut route = get_route(&our_id, &payment_params, &network_graph, None, 100, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), 100); + let mut route = get_route(&our_id, &route_params, &network_graph, None, + Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); add_random_cltv_offset(&mut route, &payment_params, &network_graph, &random_seed_bytes); let mut path_plausibility = vec![]; @@ -6047,8 +6224,9 @@ mod tests { let payment_params = PaymentParameters::from_node_id(nodes[3], 0); let hops = [nodes[1], nodes[2], nodes[4], nodes[3]]; - let route = build_route_from_hops_internal(&our_id, &hops, &payment_params, - &network_graph, 100, Arc::clone(&logger), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100); + let route = build_route_from_hops_internal(&our_id, &hops, &route_params, &network_graph, + Arc::clone(&logger), &random_seed_bytes).unwrap(); let route_hop_pubkeys = route.paths[0].hops.iter().map(|hop| hop.pubkey).collect::>(); assert_eq!(hops.len(), route.paths[0].hops.len()); for (idx, hop_pubkey) in hops.iter().enumerate() { @@ -6095,7 +6273,10 @@ mod tests { let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet); let random_seed_bytes = keys_manager.get_secure_random_bytes(); // 100,000 sats is less than the available liquidity on each channel, set above. - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100_000_000, Arc::clone(&logger), &scorer, &ProbabilisticScoringFeeParameters::default(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100_000_000); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &ProbabilisticScoringFeeParameters::default(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert!((route.paths[0].hops[1].short_channel_id == 4 && route.paths[1].hops[1].short_channel_id == 13) || (route.paths[1].hops[1].short_channel_id == 4 && route.paths[0].hops[1].short_channel_id == 13)); @@ -6196,17 +6377,22 @@ mod tests { // Then check we can get a normal route let payment_params = PaymentParameters::from_node_id(nodes[10], 42); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &scorer_params,&random_seed_bytes); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 100); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &scorer_params, &random_seed_bytes); assert!(route.is_ok()); // Then check that we can't get a route if we ban an intermediate node. scorer_params.add_banned(&NodeId::from_pubkey(&nodes[3])); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &scorer_params,&random_seed_bytes); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &scorer_params, &random_seed_bytes); assert!(route.is_err()); // Finally make sure we can route again, when we remove the ban. scorer_params.remove_banned(&NodeId::from_pubkey(&nodes[3])); - let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, Arc::clone(&logger), &scorer, &scorer_params,&random_seed_bytes); + let route = get_route(&our_id, &route_params, &network_graph.read_only(), None, + Arc::clone(&logger), &scorer, &scorer_params, &random_seed_bytes); assert!(route.is_ok()); } @@ -6241,8 +6427,10 @@ mod tests { // Make sure we'll error if our route hints don't have enough liquidity according to their // htlc_maximum_msat. + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, max_htlc_msat + 1); if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, - &payment_params, &netgraph, None, max_htlc_msat + 1, Arc::clone(&logger), &scorer, &(), + &route_params, &netgraph, None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { assert_eq!(err, "Failed to find a sufficient route to the given destination"); @@ -6254,8 +6442,10 @@ mod tests { let payment_params = PaymentParameters::from_node_id(dest_node_id, 42) .with_route_hints(vec![route_hint_1, route_hint_2]).unwrap() .with_bolt11_features(channelmanager::provided_invoice_features(&config)).unwrap(); - let route = get_route(&our_id, &payment_params, &netgraph, None, max_htlc_msat + 1, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, max_htlc_msat + 1); + let route = get_route(&our_id, &route_params, &netgraph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert!(route.paths[0].hops.last().unwrap().fee_msat <= max_htlc_msat); assert!(route.paths[1].hops.last().unwrap().fee_msat <= max_htlc_msat); @@ -6308,8 +6498,10 @@ mod tests { .with_route_hints(vec![route_hint_1, route_hint_2]).unwrap() .with_bolt11_features(channelmanager::provided_invoice_features(&config)).unwrap(); - let route = get_route(&our_node_id, &payment_params, &network_graph.read_only(), - Some(&first_hop.iter().collect::>()), amt_msat, Arc::clone(&logger), &scorer, &(), + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, amt_msat); + let route = get_route(&our_node_id, &route_params, &network_graph.read_only(), + Some(&first_hop.iter().collect::>()), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert!(route.paths[0].hops.last().unwrap().fee_msat <= max_htlc_msat); @@ -6322,8 +6514,8 @@ mod tests { get_channel_details(Some(42), intermed_node_id, InitFeatures::from_le_bytes(vec![0b11]), amt_msat - 10), get_channel_details(Some(43), intermed_node_id, InitFeatures::from_le_bytes(vec![0b11]), amt_msat - 10), ]; - let route = get_route(&our_node_id, &payment_params, &network_graph.read_only(), - Some(&first_hops.iter().collect::>()), amt_msat, Arc::clone(&logger), &scorer, &(), + let route = get_route(&our_node_id, &route_params, &network_graph.read_only(), + Some(&first_hops.iter().collect::>()), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert!(route.paths[0].hops.last().unwrap().fee_msat <= max_htlc_msat); @@ -6352,8 +6544,10 @@ mod tests { (blinded_payinfo.clone(), blinded_path.clone()), (blinded_payinfo.clone(), blinded_path.clone())]) .with_bolt12_features(bolt12_features).unwrap(); - let route = get_route(&our_node_id, &payment_params, &network_graph.read_only(), - Some(&first_hops.iter().collect::>()), amt_msat, Arc::clone(&logger), &scorer, &(), + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, amt_msat); + let route = get_route(&our_node_id, &route_params, &network_graph.read_only(), + Some(&first_hops.iter().collect::>()), Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); assert!(route.paths[0].hops.last().unwrap().fee_msat <= max_htlc_msat); @@ -6542,9 +6736,10 @@ mod tests { features: BlindedHopFeatures::empty(), }; - let final_amt_msat = 1001; let payment_params = PaymentParameters::blinded(vec![(blinded_payinfo.clone(), blinded_path.clone())]); - let route = get_route(&our_id, &payment_params, &network_graph, None, final_amt_msat , Arc::clone(&logger), + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, 1001); + let route = get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 1); assert_eq!(route.paths[0].hops.len(), 2); @@ -6598,7 +6793,8 @@ mod tests { let payment_params = PaymentParameters::blinded(vec![ (blinded_payinfo.clone(), invalid_blinded_path.clone()), (blinded_payinfo.clone(), invalid_blinded_path_2)]); - match get_route(&our_id, &payment_params, &network_graph, None, 1001, Arc::clone(&logger), + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 1001); + match get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { Err(LightningError { err, .. }) => { @@ -6609,8 +6805,9 @@ mod tests { invalid_blinded_path.introduction_node_id = our_id; let payment_params = PaymentParameters::blinded(vec![(blinded_payinfo.clone(), invalid_blinded_path.clone())]); - match get_route(&our_id, &payment_params, &network_graph, None, 1001, Arc::clone(&logger), - &scorer, &(), &random_seed_bytes) + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 1001); + match get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, + &(), &random_seed_bytes) { Err(LightningError { err, .. }) => { assert_eq!(err, "Cannot generate a route to blinded paths if we are the introduction node to all of them"); @@ -6621,8 +6818,9 @@ mod tests { invalid_blinded_path.introduction_node_id = ln_test_utils::pubkey(46); invalid_blinded_path.blinded_hops.clear(); let payment_params = PaymentParameters::blinded(vec![(blinded_payinfo, invalid_blinded_path)]); - match get_route(&our_id, &payment_params, &network_graph, None, 1001, Arc::clone(&logger), - &scorer, &(), &random_seed_bytes) + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 1001); + match get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), &scorer, + &(), &random_seed_bytes) { Err(LightningError { err, .. }) => { assert_eq!(err, "0-hop blinded path provided"); @@ -6674,8 +6872,9 @@ mod tests { let payment_params = PaymentParameters::blinded(blinded_hints.clone()) .with_bolt12_features(bolt12_features.clone()).unwrap(); - let route = get_route(&our_id, &payment_params, &network_graph, None, - 100_000, Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value(payment_params, 100_000); + let route = get_route(&our_id, &route_params, &network_graph, None, Arc::clone(&logger), + &scorer, &(), &random_seed_bytes).unwrap(); assert_eq!(route.paths.len(), 2); let mut total_amount_paid_msat = 0; for path in route.paths.into_iter() { @@ -6761,17 +6960,21 @@ mod tests { let payment_params = PaymentParameters::blinded(blinded_hints.clone()); let netgraph = network_graph.read_only(); - if let Err(LightningError { err, .. }) = get_route(&nodes[0], &payment_params, &netgraph, - Some(&first_hops.iter().collect::>()), amt_msat, Arc::clone(&logger), &scorer, &(), + let route_params = RouteParameters::from_payment_params_and_value( + payment_params.clone(), amt_msat); + if let Err(LightningError { err, .. }) = get_route(&nodes[0], &route_params, &netgraph, + Some(&first_hops.iter().collect::>()), Arc::clone(&logger), &scorer, &(), &random_seed_bytes) { - assert_eq!(err, "Failed to find a path to the given destination"); + assert_eq!(err, "Failed to find a path to the given destination"); } else { panic!("Expected error") } // Sending an exact amount accounting for the blinded path fee works. let amt_minus_blinded_path_fee = amt_msat - blinded_payinfo.fee_base_msat as u64; - let route = get_route(&nodes[0], &payment_params, &netgraph, - Some(&first_hops.iter().collect::>()), amt_minus_blinded_path_fee, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, amt_minus_blinded_path_fee); + let route = get_route(&nodes[0], &route_params, &netgraph, + Some(&first_hops.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.get_total_fees(), blinded_payinfo.fee_base_msat as u64); assert_eq!(route.get_total_amount(), amt_minus_blinded_path_fee); } @@ -6836,9 +7039,11 @@ mod tests { .with_bolt12_features(bolt12_features.clone()).unwrap(); let netgraph = network_graph.read_only(); - let route = get_route(&nodes[0], &payment_params, &netgraph, - Some(&first_hops.iter().collect::>()), amt_msat, - Arc::clone(&logger), &scorer, &(), &random_seed_bytes).unwrap(); + let route_params = RouteParameters::from_payment_params_and_value( + payment_params, amt_msat); + let route = get_route(&nodes[0], &route_params, &netgraph, + Some(&first_hops.iter().collect::>()), Arc::clone(&logger), &scorer, &(), + &random_seed_bytes).unwrap(); assert_eq!(route.get_total_fees(), blinded_payinfo.fee_base_msat as u64); assert_eq!(route.get_total_amount(), amt_msat); } @@ -6970,10 +7175,12 @@ pub(crate) mod bench_utils { let params = PaymentParameters::from_node_id(dst, 42) .with_bolt11_features(features.clone()).unwrap(); let first_hop = first_hop(src); - let amt = starting_amount + seed % 1_000_000; + let amt_msat = starting_amount + seed % 1_000_000; + let route_params = RouteParameters::from_payment_params_and_value( + params.clone(), amt_msat); let path_exists = - get_route(&payer, ¶ms, &graph.read_only(), Some(&[&first_hop]), - amt, &TestLogger::new(), scorer, score_params, &random_seed_bytes).is_ok(); + get_route(&payer, &route_params, &graph.read_only(), Some(&[&first_hop]), + &TestLogger::new(), scorer, score_params, &random_seed_bytes).is_ok(); if path_exists { // ...and seed the scorer with success and failure data... seed = seed.overflowing_mul(6364136223846793005).0.overflowing_add(1).0; @@ -6985,10 +7192,11 @@ pub(crate) mod bench_utils { let mpp_features = channelmanager::provided_invoice_features(&UserConfig::default()); let params = PaymentParameters::from_node_id(dst, 42) .with_bolt11_features(mpp_features).unwrap(); - - let route_res = get_route(&payer, ¶ms, &graph.read_only(), - Some(&[&first_hop]), score_amt, &TestLogger::new(), scorer, - score_params, &random_seed_bytes); + let route_params = RouteParameters::from_payment_params_and_value( + params.clone(), score_amt); + let route_res = get_route(&payer, &route_params, &graph.read_only(), + Some(&[&first_hop]), &TestLogger::new(), scorer, score_params, + &random_seed_bytes); if let Ok(route) = route_res { for path in route.paths { if seed & 0x80 == 0 { @@ -7005,7 +7213,7 @@ pub(crate) mod bench_utils { score_amt /= 100; } - route_endpoints.push((first_hop, params, amt)); + route_endpoints.push((first_hop, params, amt_msat)); break; } } @@ -7014,8 +7222,10 @@ pub(crate) mod bench_utils { // Because we've changed channel scores, it's possible we'll take different routes to the // selected destinations, possibly causing us to fail because, eg, the newly-selected path // requires a too-high CLTV delta. - route_endpoints.retain(|(first_hop, params, amt)| { - get_route(&payer, params, &graph.read_only(), Some(&[first_hop]), *amt, + route_endpoints.retain(|(first_hop, params, amt_msat)| { + let route_params = RouteParameters::from_payment_params_and_value( + params.clone(), *amt_msat); + get_route(&payer, &route_params, &graph.read_only(), Some(&[first_hop]), &TestLogger::new(), scorer, score_params, &random_seed_bytes).is_ok() }); route_endpoints.truncate(route_count); @@ -7106,7 +7316,8 @@ pub mod benches { let mut idx = 0; bench.bench_function(bench_name, |b| b.iter(|| { let (first_hop, params, amt) = &route_endpoints[idx % route_endpoints.len()]; - assert!(get_route(&payer, params, &graph.read_only(), Some(&[first_hop]), *amt, + let route_params = RouteParameters::from_payment_params_and_value(params.clone(), *amt); + assert!(get_route(&payer, &route_params, &graph.read_only(), Some(&[first_hop]), &DummyLogger{}, &scorer, score_params, &random_seed_bytes).is_ok()); idx += 1; })); From 976d077e0425e64c544521e9a319b5b992d01da1 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Thu, 31 Aug 2023 15:10:09 +0200 Subject: [PATCH 11/38] Have `Route` hold `RouteParameters` --- fuzz/src/chanmon_consistency.rs | 6 ++-- lightning/src/ln/functional_tests.rs | 8 +++-- lightning/src/ln/onion_utils.rs | 2 +- lightning/src/ln/outbound_payment.rs | 12 ++++---- lightning/src/ln/payment_tests.rs | 32 ++++++++++++-------- lightning/src/routing/router.rs | 37 ++++++++++++++++------- pending_changelog/routes_route_params.txt | 3 ++ 7 files changed, 65 insertions(+), 35 deletions(-) create mode 100644 pending_changelog/routes_route_params.txt diff --git a/fuzz/src/chanmon_consistency.rs b/fuzz/src/chanmon_consistency.rs index 83470bf8bc8..4c79f0bee27 100644 --- a/fuzz/src/chanmon_consistency.rs +++ b/fuzz/src/chanmon_consistency.rs @@ -376,7 +376,7 @@ fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, p fee_msat: amt, cltv_expiry_delta: 200, }], blinded_tail: None }], - payment_params: None, + route_params: None, }, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) { check_payment_err(err, amt > max_value_sendable || amt < min_value_sendable); false @@ -409,7 +409,7 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des channel_features: middle.channel_features(), fee_msat: first_hop_fee, cltv_expiry_delta: 100, - },RouteHop { + }, RouteHop { pubkey: dest.get_our_node_id(), node_features: dest.node_features(), short_channel_id: dest_chan_id, @@ -417,7 +417,7 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des fee_msat: amt, cltv_expiry_delta: 200, }], blinded_tail: None }], - payment_params: None, + route_params: None, }, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) { let sent_amt = amt + first_hop_fee; check_payment_err(err, sent_amt < min_value_sendable || sent_amt > max_value_sendable); diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 849aa042737..2b0533ae2d2 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -1049,7 +1049,9 @@ fn fake_network_test() { }); hops[1].fee_msat = chan_4.1.contents.fee_base_msat as u64 + chan_4.1.contents.fee_proportional_millionths as u64 * hops[2].fee_msat as u64 / 1000000; hops[0].fee_msat = chan_3.0.contents.fee_base_msat as u64 + chan_3.0.contents.fee_proportional_millionths as u64 * hops[1].fee_msat as u64 / 1000000; - let payment_preimage_1 = send_along_route(&nodes[1], Route { paths: vec![Path { hops, blinded_tail: None }], payment_params: None }, &vec!(&nodes[2], &nodes[3], &nodes[1])[..], 1000000).0; + let payment_preimage_1 = send_along_route(&nodes[1], + Route { paths: vec![Path { hops, blinded_tail: None }], route_params: None }, + &vec!(&nodes[2], &nodes[3], &nodes[1])[..], 1000000).0; let mut hops = Vec::with_capacity(3); hops.push(RouteHop { @@ -1078,7 +1080,9 @@ fn fake_network_test() { }); hops[1].fee_msat = chan_2.1.contents.fee_base_msat as u64 + chan_2.1.contents.fee_proportional_millionths as u64 * hops[2].fee_msat as u64 / 1000000; hops[0].fee_msat = chan_3.1.contents.fee_base_msat as u64 + chan_3.1.contents.fee_proportional_millionths as u64 * hops[1].fee_msat as u64 / 1000000; - let payment_hash_2 = send_along_route(&nodes[1], Route { paths: vec![Path { hops, blinded_tail: None }], payment_params: None }, &vec!(&nodes[3], &nodes[2], &nodes[1])[..], 1000000).1; + let payment_hash_2 = send_along_route(&nodes[1], + Route { paths: vec![Path { hops, blinded_tail: None }], route_params: None }, + &vec!(&nodes[3], &nodes[2], &nodes[1])[..], 1000000).1; // Claim the rebalances... fail_payment(&nodes[1], &vec!(&nodes[3], &nodes[2], &nodes[1])[..], payment_hash_2); diff --git a/lightning/src/ln/onion_utils.rs b/lightning/src/ln/onion_utils.rs index d55077770f3..8fdbdefef65 100644 --- a/lightning/src/ln/onion_utils.rs +++ b/lightning/src/ln/onion_utils.rs @@ -1007,7 +1007,7 @@ mod tests { short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops. }, ], blinded_tail: None }], - payment_params: None, + route_params: None, }; let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &get_test_session_key()).unwrap(); diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 38ae89aec6b..67d90d2dbf8 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -976,7 +976,7 @@ impl OutboundPayments { })) } - let route = Route { paths: vec![path], payment_params: None }; + let route = Route { paths: vec![path], route_params: None }; let onion_session_privs = self.add_new_pending_payment(payment_hash, RecipientOnionFields::spontaneous_empty(), payment_id, None, &route, None, None, entropy_source, best_block_height)?; @@ -1145,9 +1145,9 @@ impl OutboundPayments { results, payment_id, failed_paths_retry: if pending_amt_unsent != 0 { - if let Some(payment_params) = &route.payment_params { + if let Some(payment_params) = route.route_params.as_ref().map(|p| p.payment_params.clone()) { Some(RouteParameters { - payment_params: payment_params.clone(), + payment_params: payment_params, final_value_msat: pending_amt_unsent, }) } else { None } @@ -1569,7 +1569,7 @@ mod tests { let pending_events = Mutex::new(VecDeque::new()); if on_retry { outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), - PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None }, + PaymentId([0; 32]), None, &Route { paths: vec![], route_params: None }, Some(Retry::Attempts(1)), Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.retry_payment_internal( @@ -1613,7 +1613,7 @@ mod tests { let pending_events = Mutex::new(VecDeque::new()); if on_retry { outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), - PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None }, + PaymentId([0; 32]), None, &Route { paths: vec![], route_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.retry_payment_internal( @@ -1657,7 +1657,7 @@ mod tests { fee_msat: 0, cltv_expiry_delta: 0, }], blinded_tail: None }], - payment_params: Some(payment_params), + route_params: Some(route_params.clone()), }; router.expect_find_route(route_params.clone(), Ok(route.clone())); let mut route_params_w_failed_scid = route_params.clone(); diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index f8000423b56..0c94adb3560 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -94,7 +94,7 @@ fn mpp_retry() { // Initiate the MPP payment. let payment_id = PaymentId(payment_hash.0); - let mut route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); + let mut route_params = route.route_params.clone().unwrap(); nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), @@ -524,7 +524,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) { let amt_msat = 1_000_000; let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat); let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000); - let route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); + let route_params = route.route_params.unwrap().clone(); nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); check_added_monitors!(nodes[0], 1); @@ -2211,7 +2211,7 @@ fn auto_retry_partial_failure() { cltv_expiry_delta: 100, }], blinded_tail: None }, ], - payment_params: Some(route_params.payment_params.clone()), + route_params: Some(route_params.clone()), }; let retry_1_route = Route { paths: vec![ @@ -2232,7 +2232,7 @@ fn auto_retry_partial_failure() { cltv_expiry_delta: 100, }], blinded_tail: None }, ], - payment_params: Some(route_params.payment_params.clone()), + route_params: Some(route_params.clone()), }; let retry_2_route = Route { paths: vec![ @@ -2245,7 +2245,7 @@ fn auto_retry_partial_failure() { cltv_expiry_delta: 100, }], blinded_tail: None }, ], - payment_params: Some(route_params.payment_params.clone()), + route_params: Some(route_params.clone()), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route)); let mut payment_params = route_params.payment_params.clone(); @@ -2497,13 +2497,13 @@ fn retry_multi_path_single_failed_payment() { cltv_expiry_delta: 100, }], blinded_tail: None }, ], - payment_params: Some(payment_params), + route_params: Some(route_params.clone()), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); // On retry, split the payment across both channels. route.paths[0].hops[0].fee_msat = 50_000_001; route.paths[1].hops[0].fee_msat = 50_000_000; - let mut pay_params = route.payment_params.clone().unwrap(); + let mut pay_params = route.route_params.clone().unwrap().payment_params; pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap()); nodes[0].router.expect_find_route( // Note that the second request here requests the amount we originally failed to send, @@ -2578,7 +2578,9 @@ fn immediate_retry_on_failure() { cltv_expiry_delta: 100, }], blinded_tail: None }, ], - payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)), + route_params: Some(RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV), + 100_000_001)), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); // On retry, split the payment across both channels. @@ -2684,7 +2686,9 @@ fn no_extra_retries_on_back_to_back_fail() { cltv_expiry_delta: 100, }], blinded_tail: None } ], - payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)), + route_params: Some(RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV), + 100_000_000)), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); let mut second_payment_params = route_params.payment_params.clone(); @@ -2882,7 +2886,9 @@ fn test_simple_partial_retry() { cltv_expiry_delta: 100, }], blinded_tail: None } ], - payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)), + route_params: Some(RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV), + 100_000_000)), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); let mut second_payment_params = route_params.payment_params.clone(); @@ -3044,7 +3050,9 @@ fn test_threaded_payment_retries() { cltv_expiry_delta: 100, }], blinded_tail: None } ], - payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)), + route_params: Some(RouteParameters::from_payment_params_and_value( + PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV), + amt_msat - amt_msat / 1000)), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone())); @@ -3470,7 +3478,7 @@ fn test_retry_custom_tlvs() { // Initiate the payment let payment_id = PaymentId(payment_hash.0); - let mut route_params = RouteParameters::from_payment_params_and_value(route.payment_params.clone().unwrap(), amt_msat); + let mut route_params = route.route_params.clone().unwrap(); let custom_tlvs = vec![((1 << 16) + 1, vec![0x42u8; 16])]; let onion_fields = RecipientOnionFields::secret_only(payment_secret); diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 799b6407981..8b3a96eb42e 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -337,10 +337,12 @@ pub struct Route { /// [`BlindedTail`]s are present, then the pubkey of the last [`RouteHop`] in each path must be /// the same. pub paths: Vec, - /// The `payment_params` parameter passed via [`RouteParameters`] to [`find_route`]. + /// The `route_params` parameter passed to [`find_route`]. /// /// This is used by `ChannelManager` to track information which may be required for retries. - pub payment_params: Option, + /// + /// Will be `None` for objects serialized with LDK versions prior to 0.0.117. + pub route_params: Option, } impl Route { @@ -383,8 +385,11 @@ impl Writeable for Route { } } write_tlv_fields!(writer, { - (1, self.payment_params, option), + // For compatibility with LDK versions prior to 0.0.117, we take the individual + // RouteParameters' fields and reconstruct them on read. + (1, self.route_params.as_ref().map(|p| &p.payment_params), option), (2, blinded_tails, optional_vec), + (3, self.route_params.as_ref().map(|p| p.final_value_msat), option), }); Ok(()) } @@ -411,6 +416,7 @@ impl Readable for Route { _init_and_read_len_prefixed_tlv_fields!(reader, { (1, payment_params, (option: ReadableArgs, min_final_cltv_expiry_delta)), (2, blinded_tails, optional_vec), + (3, final_value_msat, option), }); let blinded_tails = blinded_tails.unwrap_or(Vec::new()); if blinded_tails.len() != 0 { @@ -419,14 +425,23 @@ impl Readable for Route { path.blinded_tail = blinded_tail_opt; } } - Ok(Route { paths, payment_params }) + + // If we previously wrote the corresponding fields, reconstruct RouteParameters. + let route_params = match (payment_params, final_value_msat) { + (Some(payment_params), Some(final_value_msat)) => { + Some(RouteParameters { payment_params, final_value_msat }) + } + _ => None, + }; + + Ok(Route { paths, route_params }) } } /// Parameters needed to find a [`Route`]. /// /// Passed to [`find_route`] and [`build_route_from_hops`]. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct RouteParameters { /// The parameters of the failed payment path. pub payment_params: PaymentParameters, @@ -2489,7 +2504,7 @@ where L::Target: Logger { } } - let route = Route { paths, payment_params: Some(payment_params.clone()) }; + let route = Route { paths, route_params: Some(route_params.clone()) }; log_info!(logger, "Got route: {}", log_route!(route)); Ok(route) } @@ -5953,7 +5968,7 @@ mod tests { short_channel_id: 0, fee_msat: 225, cltv_expiry_delta: 0 }, ], blinded_tail: None }], - payment_params: None, + route_params: None, }; assert_eq!(route.get_total_fees(), 250); @@ -5986,7 +6001,7 @@ mod tests { short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0 }, ], blinded_tail: None }], - payment_params: None, + route_params: None, }; assert_eq!(route.get_total_fees(), 200); @@ -5998,7 +6013,7 @@ mod tests { // In an earlier version of `Route::get_total_fees` and `Route::get_total_amount`, they // would both panic if the route was completely empty. We test to ensure they return 0 // here, even though its somewhat nonsensical as a route. - let route = Route { paths: Vec::new(), payment_params: None }; + let route = Route { paths: Vec::new(), route_params: None }; assert_eq!(route.get_total_fees(), 0); assert_eq!(route.get_total_amount(), 0); @@ -6597,7 +6612,7 @@ mod tests { fee_msat: 100, cltv_expiry_delta: 0, }], blinded_tail: None }], - payment_params: None, + route_params: None, }; let encoded_route = route.encode(); let decoded_route: Route = Readable::read(&mut Cursor::new(&encoded_route[..])).unwrap(); @@ -6691,7 +6706,7 @@ mod tests { excess_final_cltv_expiry_delta: 0, final_value_msat: 200, }), - }], payment_params: None}; + }], route_params: None}; let payment_params = PaymentParameters::from_node_id(ln_test_utils::pubkey(47), 18); let (_, network_graph, _, _, _) = build_line_graph(); diff --git a/pending_changelog/routes_route_params.txt b/pending_changelog/routes_route_params.txt new file mode 100644 index 00000000000..e88a1c78116 --- /dev/null +++ b/pending_changelog/routes_route_params.txt @@ -0,0 +1,3 @@ +# Backwards Compatibility + +- `Route` objects written with LDK versions prior to 0.0.117 won't be retryable after being deserialized with LDK 0.0.117 or above. From 539d60dec3184f0744bcf297c1351f2e00babba1 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 6 Sep 2023 11:17:51 +0200 Subject: [PATCH 12/38] Include overpaid value in `Route::get_total_fees` --- lightning/src/routing/router.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 8b3a96eb42e..10bdd69b51a 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -348,10 +348,15 @@ pub struct Route { impl Route { /// Returns the total amount of fees paid on this [`Route`]. /// - /// This doesn't include any extra payment made to the recipient, which can happen in excess of - /// the amount passed to [`find_route`]'s `route_params.final_value_msat`. + /// For objects serialized with LDK 0.0.117 and after, this includes any extra payment made to + /// the recipient, which can happen in excess of the amount passed to [`find_route`] via + /// [`RouteParameters::final_value_msat`], if we had to reach the [`htlc_minimum_msat`] limits. + /// + /// [`htlc_minimum_msat`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_update-message pub fn get_total_fees(&self) -> u64 { - self.paths.iter().map(|path| path.fee_msat()).sum() + let overpaid_value_msat = self.route_params.as_ref() + .map_or(0, |p| self.get_total_amount().saturating_sub(p.final_value_msat)); + overpaid_value_msat + self.paths.iter().map(|path| path.fee_msat()).sum::() } /// Returns the total amount paid on this [`Route`], excluding the fees. Might be more than From 9963e161123675121af03176f508bf49b6a66caf Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 6 Sep 2023 11:26:13 +0200 Subject: [PATCH 13/38] Improve `Route::get_total_amount` docs --- lightning/src/routing/router.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 10bdd69b51a..9c5fe8e1f9b 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -359,8 +359,12 @@ impl Route { overpaid_value_msat + self.paths.iter().map(|path| path.fee_msat()).sum::() } - /// Returns the total amount paid on this [`Route`], excluding the fees. Might be more than - /// requested if we had to reach htlc_minimum_msat. + /// Returns the total amount paid on this [`Route`], excluding the fees. + /// + /// Might be more than requested as part of the given [`RouteParameters::final_value_msat`] if + /// we had to reach the [`htlc_minimum_msat`] limits. + /// + /// [`htlc_minimum_msat`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_update-message pub fn get_total_amount(&self) -> u64 { self.paths.iter().map(|path| path.final_value_msat()).sum() } From 40d901fb2dcafd2acff1abfa54947c928ba86759 Mon Sep 17 00:00:00 2001 From: Chris Waterson Date: Wed, 6 Sep 2023 08:38:08 -0700 Subject: [PATCH 14/38] Log each condition that was violated for a stale monitor There are several conditions that can be violated which indicate a stale monitor. This logs each that doesn't hold. --- lightning/src/ln/channelmanager.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index cff9837fceb..b70c160f391 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -8637,8 +8637,22 @@ where // But if the channel is behind of the monitor, close the channel: log_error!(args.logger, "A ChannelManager is stale compared to the current ChannelMonitor!"); log_error!(args.logger, " The channel will be force-closed and the latest commitment transaction from the ChannelMonitor broadcast."); - log_error!(args.logger, " The ChannelMonitor for channel {} is at update_id {} but the ChannelManager is at update_id {}.", - &channel.context.channel_id(), monitor.get_latest_update_id(), channel.context.get_latest_monitor_update_id()); + if channel.context.get_latest_monitor_update_id() < monitor.get_latest_update_id() { + log_error!(args.logger, " The ChannelMonitor for channel {} is at update_id {} and the ChannelManager is at update_id {}.", + &channel.context.channel_id(), monitor.get_latest_update_id(), channel.context.get_latest_monitor_update_id()); + } + if channel.get_cur_holder_commitment_transaction_number() > monitor.get_cur_holder_commitment_number() { + log_error!(args.logger, " The ChannelMonitor for channel {} is at holder commitment number {} and the ChannelManager is at holder commitment {}.", + &channel.context.channel_id(), monitor.get_cur_holder_commitment_number(), channel.get_cur_holder_commitment_transaction_number()); + } + if channel.get_revoked_counterparty_commitment_transaction_number() > monitor.get_min_seen_secret() { + log_error!(args.logger, " The ChannelMonitor for channel {} is at revoked counterparty transaction number {} but the ChannelManager is at revoked counterparty transaction number {}.", + &channel.context.channel_id(), monitor.get_min_seen_secret(), channel.get_revoked_counterparty_commitment_transaction_number()); + } + if channel.get_cur_counterparty_commitment_transaction_number() > monitor.get_cur_counterparty_commitment_number() { + log_error!(args.logger, " The ChannelMonitor for channel {} is at counterparty commitment transaction number {} but the ChannelManager is at counterparty commtiment transaction number {}.", + &channel.context.channel_id(), monitor.get_cur_counterparty_commitment_number(), channel.get_cur_counterparty_commitment_transaction_number()); + } let (monitor_update, mut new_failed_htlcs) = channel.context.force_shutdown(true); if let Some((counterparty_node_id, funding_txo, update)) = monitor_update { close_background_events.push(BackgroundEvent::MonitorUpdateRegeneratedOnStartup { From b874ccfd6f63ab199b8b96090ba65fc8ccfd3e2e Mon Sep 17 00:00:00 2001 From: Chris Waterson Date: Wed, 6 Sep 2023 15:10:51 -0700 Subject: [PATCH 15/38] Fix misspelling, s/and/but/. --- lightning/src/ln/channelmanager.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index b70c160f391..5f4ea2969f9 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -8638,11 +8638,11 @@ where log_error!(args.logger, "A ChannelManager is stale compared to the current ChannelMonitor!"); log_error!(args.logger, " The channel will be force-closed and the latest commitment transaction from the ChannelMonitor broadcast."); if channel.context.get_latest_monitor_update_id() < monitor.get_latest_update_id() { - log_error!(args.logger, " The ChannelMonitor for channel {} is at update_id {} and the ChannelManager is at update_id {}.", + log_error!(args.logger, " The ChannelMonitor for channel {} is at update_id {} but the ChannelManager is at update_id {}.", &channel.context.channel_id(), monitor.get_latest_update_id(), channel.context.get_latest_monitor_update_id()); } if channel.get_cur_holder_commitment_transaction_number() > monitor.get_cur_holder_commitment_number() { - log_error!(args.logger, " The ChannelMonitor for channel {} is at holder commitment number {} and the ChannelManager is at holder commitment {}.", + log_error!(args.logger, " The ChannelMonitor for channel {} is at holder commitment number {} but the ChannelManager is at holder commitment number {}.", &channel.context.channel_id(), monitor.get_cur_holder_commitment_number(), channel.get_cur_holder_commitment_transaction_number()); } if channel.get_revoked_counterparty_commitment_transaction_number() > monitor.get_min_seen_secret() { @@ -8650,7 +8650,7 @@ where &channel.context.channel_id(), monitor.get_min_seen_secret(), channel.get_revoked_counterparty_commitment_transaction_number()); } if channel.get_cur_counterparty_commitment_transaction_number() > monitor.get_cur_counterparty_commitment_number() { - log_error!(args.logger, " The ChannelMonitor for channel {} is at counterparty commitment transaction number {} but the ChannelManager is at counterparty commtiment transaction number {}.", + log_error!(args.logger, " The ChannelMonitor for channel {} is at counterparty commitment transaction number {} but the ChannelManager is at counterparty commitment transaction number {}.", &channel.context.channel_id(), monitor.get_cur_counterparty_commitment_number(), channel.get_cur_counterparty_commitment_transaction_number()); } let (monitor_update, mut new_failed_htlcs) = channel.context.force_shutdown(true); From eb593db804ec55976bfba18686d922550262e45d Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Thu, 7 Sep 2023 11:48:37 +0530 Subject: [PATCH 16/38] logging channel_id of connected txids --- lightning/src/ln/channelmanager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 5f4ea2969f9..14b3fd716bb 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -7326,6 +7326,7 @@ where fn handle_channel_ready(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelReady) { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); let _ = handle_error!(self, self.internal_channel_ready(counterparty_node_id, msg), *counterparty_node_id); + log_trace!(self.logger, "connected txids Channel_id {}", msg.channel_id); } fn handle_shutdown(&self, counterparty_node_id: &PublicKey, msg: &msgs::Shutdown) { From 62afba67d88444d20fd8126145796d81d5e72d51 Mon Sep 17 00:00:00 2001 From: Lalitmohansharma1 Date: Sat, 9 Sep 2023 16:00:42 +0530 Subject: [PATCH 17/38] logging confirmed txid --- lightning/src/ln/channelmanager.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 14b3fd716bb..d620d03e426 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -6992,6 +6992,12 @@ where let timestamp = self.highest_seen_timestamp.load(Ordering::Acquire); self.do_chain_event(Some(last_best_block_height), |channel| channel.best_block_updated(last_best_block_height, timestamp as u32, self.genesis_hash.clone(), &self.node_signer, &self.default_configuration, &self.logger)); } + // log each confirmed tansaction's Txid. + for tx in txdata.iter() { + let (index, transaction) = tx; + let txid = transaction.txid(); + log_trace!(self.logger, "Transaction id {} confirmed in block {}", txid, block_hash); + } } fn best_block_updated(&self, header: &BlockHeader, height: u32) { @@ -7326,7 +7332,6 @@ where fn handle_channel_ready(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelReady) { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); let _ = handle_error!(self, self.internal_channel_ready(counterparty_node_id, msg), *counterparty_node_id); - log_trace!(self.logger, "connected txids Channel_id {}", msg.channel_id); } fn handle_shutdown(&self, counterparty_node_id: &PublicKey, msg: &msgs::Shutdown) { From e0523244c5a534d417a400a95c5d594eca9cd4ad Mon Sep 17 00:00:00 2001 From: jbesraa Date: Fri, 25 Aug 2023 05:52:38 +0300 Subject: [PATCH 18/38] Add RFC4648 base32 `encode` and `decode` functions --- fuzz/src/base32.rs | 52 ++++++++ fuzz/src/bin/base32_target.rs | 113 ++++++++++++++++++ fuzz/src/bin/gen_target.sh | 1 + fuzz/src/lib.rs | 1 + fuzz/targets.h | 1 + lightning/src/util/base32.rs | 218 ++++++++++++++++++++++++++++++++++ lightning/src/util/mod.rs | 4 + 7 files changed, 390 insertions(+) create mode 100644 fuzz/src/base32.rs create mode 100644 fuzz/src/bin/base32_target.rs create mode 100644 lightning/src/util/base32.rs diff --git a/fuzz/src/base32.rs b/fuzz/src/base32.rs new file mode 100644 index 00000000000..8171f19f637 --- /dev/null +++ b/fuzz/src/base32.rs @@ -0,0 +1,52 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +use lightning::util::base32; + +use crate::utils::test_logger; + +#[inline] +pub fn do_test(data: &[u8]) { + if let Ok(s) = std::str::from_utf8(data) { + let first_decoding = base32::Alphabet::RFC4648 { padding: true }.decode(s); + if let Ok(first_decoding) = first_decoding { + let encoding_response = base32::Alphabet::RFC4648 { padding: true }.encode(&first_decoding); + assert_eq!(encoding_response, s.to_ascii_uppercase()); + let second_decoding = base32::Alphabet::RFC4648 { padding: true }.decode(&encoding_response).unwrap(); + assert_eq!(first_decoding, second_decoding); + } + } + + if let Ok(s) = std::str::from_utf8(data) { + let first_decoding = base32::Alphabet::RFC4648 { padding: false }.decode(s); + if let Ok(first_decoding) = first_decoding { + let encoding_response = base32::Alphabet::RFC4648 { padding: false }.encode(&first_decoding); + assert_eq!(encoding_response, s.to_ascii_uppercase()); + let second_decoding = base32::Alphabet::RFC4648 { padding: false }.decode(&encoding_response).unwrap(); + assert_eq!(first_decoding, second_decoding); + } + } + + let encode_response = base32::Alphabet::RFC4648 { padding: false }.encode(&data); + let decode_response = base32::Alphabet::RFC4648 { padding: false }.decode(&encode_response).unwrap(); + assert_eq!(data, decode_response); + + let encode_response = base32::Alphabet::RFC4648 { padding: true }.encode(&data); + let decode_response = base32::Alphabet::RFC4648 { padding: true }.decode(&encode_response).unwrap(); + assert_eq!(data, decode_response); +} + +pub fn base32_test(data: &[u8], _out: Out) { + do_test(data); +} + +#[no_mangle] +pub extern "C" fn base32_run(data: *const u8, datalen: usize) { + do_test(unsafe { std::slice::from_raw_parts(data, datalen) }); +} diff --git a/fuzz/src/bin/base32_target.rs b/fuzz/src/bin/base32_target.rs new file mode 100644 index 00000000000..a7951c77004 --- /dev/null +++ b/fuzz/src/bin/base32_target.rs @@ -0,0 +1,113 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +// This file is auto-generated by gen_target.sh based on target_template.txt +// To modify it, modify target_template.txt and run gen_target.sh instead. + +#![cfg_attr(feature = "libfuzzer_fuzz", no_main)] + +#[cfg(not(fuzzing))] +compile_error!("Fuzz targets need cfg=fuzzing"); + +extern crate lightning_fuzz; +use lightning_fuzz::base32::*; + +#[cfg(feature = "afl")] +#[macro_use] extern crate afl; +#[cfg(feature = "afl")] +fn main() { + fuzz!(|data| { + base32_run(data.as_ptr(), data.len()); + }); +} + +#[cfg(feature = "honggfuzz")] +#[macro_use] extern crate honggfuzz; +#[cfg(feature = "honggfuzz")] +fn main() { + loop { + fuzz!(|data| { + base32_run(data.as_ptr(), data.len()); + }); + } +} + +#[cfg(feature = "libfuzzer_fuzz")] +#[macro_use] extern crate libfuzzer_sys; +#[cfg(feature = "libfuzzer_fuzz")] +fuzz_target!(|data: &[u8]| { + base32_run(data.as_ptr(), data.len()); +}); + +#[cfg(feature = "stdin_fuzz")] +fn main() { + use std::io::Read; + + let mut data = Vec::with_capacity(8192); + std::io::stdin().read_to_end(&mut data).unwrap(); + base32_run(data.as_ptr(), data.len()); +} + +#[test] +fn run_test_cases() { + use std::fs; + use std::io::Read; + use lightning_fuzz::utils::test_logger::StringBuffer; + + use std::sync::{atomic, Arc}; + { + let data: Vec = vec![0]; + base32_run(data.as_ptr(), data.len()); + } + let mut threads = Vec::new(); + let threads_running = Arc::new(atomic::AtomicUsize::new(0)); + if let Ok(tests) = fs::read_dir("test_cases/base32") { + for test in tests { + let mut data: Vec = Vec::new(); + let path = test.unwrap().path(); + fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap(); + threads_running.fetch_add(1, atomic::Ordering::AcqRel); + + let thread_count_ref = Arc::clone(&threads_running); + let main_thread_ref = std::thread::current(); + threads.push((path.file_name().unwrap().to_str().unwrap().to_string(), + std::thread::spawn(move || { + let string_logger = StringBuffer::new(); + + let panic_logger = string_logger.clone(); + let res = if ::std::panic::catch_unwind(move || { + base32_test(&data, panic_logger); + }).is_err() { + Some(string_logger.into_string()) + } else { None }; + thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel); + main_thread_ref.unpark(); + res + }) + )); + while threads_running.load(atomic::Ordering::Acquire) > 32 { + std::thread::park(); + } + } + } + let mut failed_outputs = Vec::new(); + for (test, thread) in threads.drain(..) { + if let Some(output) = thread.join().unwrap() { + println!("\nOutput of {}:\n{}\n", test, output); + failed_outputs.push(test); + } + } + if !failed_outputs.is_empty() { + println!("Test cases which failed: "); + for case in failed_outputs { + println!("{}", case); + } + panic!(); + } +} diff --git a/fuzz/src/bin/gen_target.sh b/fuzz/src/bin/gen_target.sh index fe17e4bab8f..676bfb82156 100755 --- a/fuzz/src/bin/gen_target.sh +++ b/fuzz/src/bin/gen_target.sh @@ -21,6 +21,7 @@ GEN_TEST router GEN_TEST zbase32 GEN_TEST indexedmap GEN_TEST onion_hop_data +GEN_TEST base32 GEN_TEST msg_accept_channel msg_targets:: GEN_TEST msg_announcement_signatures msg_targets:: diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 6cdeb8ab5d2..607924eff95 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -29,5 +29,6 @@ pub mod refund_deser; pub mod router; pub mod zbase32; pub mod onion_hop_data; +pub mod base32; pub mod msg_targets; diff --git a/fuzz/targets.h b/fuzz/targets.h index 9b5a6d45536..a17231c6d6d 100644 --- a/fuzz/targets.h +++ b/fuzz/targets.h @@ -14,6 +14,7 @@ void router_run(const unsigned char* data, size_t data_len); void zbase32_run(const unsigned char* data, size_t data_len); void indexedmap_run(const unsigned char* data, size_t data_len); void onion_hop_data_run(const unsigned char* data, size_t data_len); +void base32_run(const unsigned char* data, size_t data_len); void msg_accept_channel_run(const unsigned char* data, size_t data_len); void msg_announcement_signatures_run(const unsigned char* data, size_t data_len); void msg_channel_reestablish_run(const unsigned char* data, size_t data_len); diff --git a/lightning/src/util/base32.rs b/lightning/src/util/base32.rs new file mode 100644 index 00000000000..ff30a024f54 --- /dev/null +++ b/lightning/src/util/base32.rs @@ -0,0 +1,218 @@ +// This is a modification of base32 encoding to support the zbase32 alphabet. +// The original piece of software can be found at https://crates.io/crates/base32(v0.4.0) +// The original portions of this software are Copyright (c) 2015 The base32 Developers + +// This file is licensed under either of +// Apache License, Version 2.0, (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or +// MIT license (LICENSE-MIT or http://opensource.org/licenses/MIT) at your option. + + +use crate::prelude::*; + +/// RFC4648 encoding table +const RFC4648_ALPHABET: &'static [u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +/// RFC4648 decoding table +const RFC4648_INV_ALPHABET: [i8; 43] = [ + -1, -1, 26, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, +]; + +/// Alphabet used for encoding and decoding. +#[derive(Copy, Clone)] +pub enum Alphabet { + /// RFC4648 encoding. + RFC4648 { + /// Whether to use padding. + padding: bool + } +} + +impl Alphabet { + /// Encode bytes into a base32 string. + pub fn encode(&self, data: &[u8]) -> String { + // output_length is calculated as follows: + // / 5 divides the data length by the number of bits per chunk (5), + // * 8 multiplies the result by the number of characters per chunk (8). + // + 4 rounds up to the nearest character. + let output_length = (data.len() * 8 + 4) / 5; + let mut ret = match self { + Self::RFC4648 { padding } => { + let mut ret = Self::encode_data(data, RFC4648_ALPHABET); + if *padding { + let len = ret.len(); + for i in output_length..len { + ret[i] = b'='; + } + + return String::from_utf8(ret).expect("Invalid UTF-8"); + } + ret + } + }; + ret.truncate(output_length); + + #[cfg(fuzzing)] + assert_eq!(ret.capacity(), (data.len() + 4) / 5 * 8); + + String::from_utf8(ret).expect("Invalid UTF-8") + } + + /// Decode a base32 string into a byte vector. + pub fn decode(&self, data: &str) -> Result, ()> { + let data = data.as_bytes(); + let (data, alphabet) = match self { + Self::RFC4648 { padding } => { + let mut unpadded_data_length = data.len(); + if *padding { + if data.len() % 8 != 0 { return Err(()); } + data.iter().rev().take(6).for_each(|&c| { + if c == b'=' { + unpadded_data_length -= 1; + } + }); + } + (&data[..unpadded_data_length], RFC4648_INV_ALPHABET) + } + }; + // If the string has more characters than are required to alphabet_encode the number of bytes + // decodable, treat the string as invalid. + match data.len() % 8 { 1|3|6 => return Err(()), _ => {} } + Ok(Self::decode_data(data, alphabet)?) + } + + /// Encode a byte slice into a base32 string. + fn encode_data(data: &[u8], alphabet: &'static [u8]) -> Vec { + // cap is calculated as follows: + // / 5 divides the data length by the number of bits per chunk (5), + // * 8 multiplies the result by the number of characters per chunk (8). + // + 4 rounds up to the nearest character. + let cap = (data.len() + 4) / 5 * 8; + let mut ret = Vec::with_capacity(cap); + for chunk in data.chunks(5) { + let mut buf = [0u8; 5]; + for (i, &b) in chunk.iter().enumerate() { + buf[i] = b; + } + ret.push(alphabet[((buf[0] & 0xF8) >> 3) as usize]); + ret.push(alphabet[(((buf[0] & 0x07) << 2) | ((buf[1] & 0xC0) >> 6)) as usize]); + ret.push(alphabet[((buf[1] & 0x3E) >> 1) as usize]); + ret.push(alphabet[(((buf[1] & 0x01) << 4) | ((buf[2] & 0xF0) >> 4)) as usize]); + ret.push(alphabet[(((buf[2] & 0x0F) << 1) | (buf[3] >> 7)) as usize]); + ret.push(alphabet[((buf[3] & 0x7C) >> 2) as usize]); + ret.push(alphabet[(((buf[3] & 0x03) << 3) | ((buf[4] & 0xE0) >> 5)) as usize]); + ret.push(alphabet[(buf[4] & 0x1F) as usize]); + } + #[cfg(fuzzing)] + assert_eq!(ret.capacity(), cap); + + ret + } + + fn decode_data(data: &[u8], alphabet: [i8; 43]) -> Result, ()> { + // cap is calculated as follows: + // / 8 divides the data length by the number of characters per chunk (8), + // * 5 multiplies the result by the number of bits per chunk (5), + // + 7 rounds up to the nearest byte. + let cap = (data.len() + 7) / 8 * 5; + let mut ret = Vec::with_capacity(cap); + for chunk in data.chunks(8) { + let mut buf = [0u8; 8]; + for (i, &c) in chunk.iter().enumerate() { + match alphabet.get(c.to_ascii_uppercase().wrapping_sub(b'0') as usize) { + Some(&-1) | None => return Err(()), + Some(&value) => buf[i] = value as u8, + }; + } + ret.push((buf[0] << 3) | (buf[1] >> 2)); + ret.push((buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4)); + ret.push((buf[3] << 4) | (buf[4] >> 1)); + ret.push((buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3)); + ret.push((buf[6] << 5) | buf[7]); + } + let output_length = data.len() * 5 / 8; + for c in ret.drain(output_length..) { + if c != 0 { + // If the original string had any bits set at positions outside of the encoded data, + // treat the string as invalid. + return Err(()); + } + } + + // Check that our capacity calculation doesn't under-shoot in fuzzing + #[cfg(fuzzing)] + assert_eq!(ret.capacity(), cap); + Ok(ret) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RFC4648_NON_PADDED_TEST_VECTORS: &[(&[u8], &[u8])] = &[ + (&[0xF8, 0x3E, 0x7F, 0x83, 0xE7], b"7A7H7A7H"), + (&[0x77, 0xC1, 0xF7, 0x7C, 0x1F], b"O7A7O7A7"), + (&[0xF8, 0x3E, 0x7F, 0x83, 0xE7], b"7A7H7A7H"), + (&[0x77, 0xC1, 0xF7, 0x7C, 0x1F], b"O7A7O7A7"), + ]; + + const RFC4648_TEST_VECTORS: &[(&[u8], &str)] = &[ + (b"", ""), + (b"f", "MY======"), + (b"fo", "MZXQ===="), + (b"foo", "MZXW6==="), + (b"foob", "MZXW6YQ="), + (b"fooba", "MZXW6YTB"), + (b"foobar", "MZXW6YTBOI======"), + (&[0xF8, 0x3E, 0x7F, 0x83], "7A7H7AY="), + ]; + + #[test] + fn test_rfc4648_encode() { + for (input, encoded) in RFC4648_TEST_VECTORS { + assert_eq!(&Alphabet::RFC4648 { padding: true }.encode(input), encoded); + } + + for (input, encoded) in RFC4648_NON_PADDED_TEST_VECTORS { + assert_eq!(&Alphabet::RFC4648 { padding: false }.encode(input).as_bytes(), encoded); + } + } + + #[test] + fn test_rfc4648_decode() { + for (input, encoded) in RFC4648_TEST_VECTORS { + let res = &Alphabet::RFC4648 { padding: true }.decode(encoded).unwrap(); + assert_eq!(&res[..], &input[..]); + } + + for (input, encoded) in RFC4648_NON_PADDED_TEST_VECTORS { + let res = &Alphabet::RFC4648 { padding: false }.decode(std::str::from_utf8(encoded).unwrap()).unwrap(); + assert_eq!(&res[..], &input[..]); + } + } + + #[test] + fn padding() { + let num_padding = [0, 6, 4, 3, 1]; + for i in 1..6 { + let encoded = Alphabet::RFC4648 { padding: true }.encode( + (0..(i as u8)).collect::>().as_ref() + ); + assert_eq!(encoded.len(), 8); + for j in 0..(num_padding[i % 5]) { + assert_eq!(encoded.as_bytes()[encoded.len() - j - 1], b'='); + } + for j in 0..(8 - num_padding[i % 5]) { + assert!(encoded.as_bytes()[j] != b'='); + } + } + } + + #[test] + fn test_decode_rfc4648_errors() { + assert!(Alphabet::RFC4648 { padding: false }.decode("abc2def===").is_err()); // Invalid char because padding is disabled + assert!(Alphabet::RFC4648 { padding: true }.decode("abc2def===").is_err()); // Invalid length + assert!(Alphabet::RFC4648 { padding: true }.decode("MZX=6YTB").is_err()); // Invalid char + } +} diff --git a/lightning/src/util/mod.rs b/lightning/src/util/mod.rs index cc1b5f581af..7eace221779 100644 --- a/lightning/src/util/mod.rs +++ b/lightning/src/util/mod.rs @@ -22,6 +22,10 @@ pub mod invoice; pub mod persist; pub mod string; pub mod wakers; +#[cfg(fuzzing)] +pub mod base32; +#[cfg(not(fuzzing))] +pub(crate) mod base32; pub(crate) mod atomic_counter; pub(crate) mod byte_utils; From 84e3ba662fe4fbea9e3511dfd3a8adb0d402ca38 Mon Sep 17 00:00:00 2001 From: jbesraa Date: Fri, 25 Aug 2023 05:57:06 +0300 Subject: [PATCH 19/38] Move `zbase32` implementation to `base32` file --- fuzz/src/zbase32.rs | 11 +- lightning/src/util/base32.rs | 59 ++++++++++- lightning/src/util/message_signing.rs | 8 +- lightning/src/util/mod.rs | 4 - lightning/src/util/zbase32.rs | 144 -------------------------- 5 files changed, 67 insertions(+), 159 deletions(-) delete mode 100644 lightning/src/util/zbase32.rs diff --git a/fuzz/src/zbase32.rs b/fuzz/src/zbase32.rs index 5ea453bfb86..04979204e98 100644 --- a/fuzz/src/zbase32.rs +++ b/fuzz/src/zbase32.rs @@ -7,18 +7,19 @@ // You may not use this file except in accordance with one or both of these // licenses. -use lightning::util::zbase32; +use lightning::util::base32; use crate::utils::test_logger; #[inline] pub fn do_test(data: &[u8]) { - let res = zbase32::encode(data); - assert_eq!(&zbase32::decode(&res).unwrap()[..], data); + let res = base32::Alphabet::ZBase32.encode(data); + assert_eq!(&base32::Alphabet::ZBase32.decode(&res).unwrap()[..], data); if let Ok(s) = std::str::from_utf8(data) { - if let Ok(decoded) = zbase32::decode(s) { - assert_eq!(&zbase32::encode(&decoded), &s.to_ascii_lowercase()); + let res = base32::Alphabet::ZBase32.decode(s); + if let Ok(decoded) = res { + assert_eq!(&base32::Alphabet::ZBase32.encode(&decoded), &s.to_ascii_lowercase()); } } } diff --git a/lightning/src/util/base32.rs b/lightning/src/util/base32.rs index ff30a024f54..2e66d59383f 100644 --- a/lightning/src/util/base32.rs +++ b/lightning/src/util/base32.rs @@ -12,12 +12,21 @@ use crate::prelude::*; /// RFC4648 encoding table const RFC4648_ALPHABET: &'static [u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; +/// Zbase encoding alphabet +const ZBASE_ALPHABET: &'static [u8] = b"ybndrfg8ejkmcpqxot1uwisza345h769"; + /// RFC4648 decoding table const RFC4648_INV_ALPHABET: [i8; 43] = [ -1, -1, 26, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, ]; +/// Zbase decoding table +const ZBASE_INV_ALPHABET: [i8; 43] = [ + -1, 18, -1, 25, 26, 27, 30, 29, 7, 31, -1, -1, -1, -1, -1, -1, -1, 24, 1, 12, 3, 8, 5, 6, 28, + 21, 9, 10, -1, 11, 2, 16, 13, 14, 4, 22, 17, 19, -1, 20, 15, 0, 23, +]; + /// Alphabet used for encoding and decoding. #[derive(Copy, Clone)] pub enum Alphabet { @@ -25,7 +34,9 @@ pub enum Alphabet { RFC4648 { /// Whether to use padding. padding: bool - } + }, + /// Zbase32 encoding. + ZBase32 } impl Alphabet { @@ -48,7 +59,10 @@ impl Alphabet { return String::from_utf8(ret).expect("Invalid UTF-8"); } ret - } + }, + Self::ZBase32 => { + Self::encode_data(data, ZBASE_ALPHABET) + }, }; ret.truncate(output_length); @@ -73,6 +87,9 @@ impl Alphabet { }); } (&data[..unpadded_data_length], RFC4648_INV_ALPHABET) + }, + Self::ZBase32 => { + (data, ZBASE_INV_ALPHABET) } }; // If the string has more characters than are required to alphabet_encode the number of bytes @@ -150,6 +167,44 @@ impl Alphabet { mod tests { use super::*; + const ZBASE32_TEST_DATA: &[(&str, &[u8])] = &[ + ("", &[]), + ("yy", &[0x00]), + ("oy", &[0x80]), + ("tqrey", &[0x8b, 0x88, 0x80]), + ("6n9hq", &[0xf0, 0xbf, 0xc7]), + ("4t7ye", &[0xd4, 0x7a, 0x04]), + ("6im5sdy", &[0xf5, 0x57, 0xbb, 0x0c]), + ("ybndrfg8ejkmcpqxot1uwisza345h769", &[0x00, 0x44, 0x32, 0x14, 0xc7, 0x42, 0x54, 0xb6, + 0x35, 0xcf, 0x84, 0x65, 0x3a, 0x56, 0xd7, 0xc6, + 0x75, 0xbe, 0x77, 0xdf]) + ]; + + #[test] + fn test_zbase32_encode() { + for &(zbase32, data) in ZBASE32_TEST_DATA { + assert_eq!(Alphabet::ZBase32.encode(data), zbase32); + } + } + + #[test] + fn test_zbase32_decode() { + for &(zbase32, data) in ZBASE32_TEST_DATA { + assert_eq!(Alphabet::ZBase32.decode(zbase32).unwrap(), data); + } + } + + #[test] + fn test_decode_wrong() { + const WRONG_DATA: &[&str] = &["00", "l1", "?", "="]; + for &data in WRONG_DATA { + match Alphabet::ZBase32.decode(data) { + Ok(_) => assert!(false, "Data shouldn't be decodable"), + Err(_) => assert!(true), + } + } + } + const RFC4648_NON_PADDED_TEST_VECTORS: &[(&[u8], &[u8])] = &[ (&[0xF8, 0x3E, 0x7F, 0x83, 0xE7], b"7A7H7A7H"), (&[0x77, 0xC1, 0xF7, 0x7C, 0x1F], b"O7A7O7A7"), diff --git a/lightning/src/util/message_signing.rs b/lightning/src/util/message_signing.rs index aa1fdfcee15..4a159fc6327 100644 --- a/lightning/src/util/message_signing.rs +++ b/lightning/src/util/message_signing.rs @@ -21,7 +21,7 @@ //! use crate::prelude::*; -use crate::util::zbase32; +use crate::util::base32; use bitcoin::hashes::{sha256d, Hash}; use bitcoin::secp256k1::ecdsa::{RecoverableSignature, RecoveryId}; use bitcoin::secp256k1::{Error, Message, PublicKey, Secp256k1, SecretKey}; @@ -58,7 +58,7 @@ pub fn sign(msg: &[u8], sk: &SecretKey) -> Result { let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); let sig = secp_ctx.sign_ecdsa_recoverable(&Message::from_slice(&msg_hash)?, sk); - Ok(zbase32::encode(&sigrec_encode(sig))) + Ok(base32::Alphabet::ZBase32.encode(&sigrec_encode(sig))) } /// Recovers the PublicKey of the signer of the message given the message and the signature. @@ -66,7 +66,7 @@ pub fn recover_pk(msg: &[u8], sig: &str) -> Result { let secp_ctx = Secp256k1::verification_only(); let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); - match zbase32::decode(&sig) { + match base32::Alphabet::ZBase32.decode(&sig) { Ok(sig_rec) => { match sigrec_decode(sig_rec) { Ok(sig) => secp_ctx.recover_ecdsa(&Message::from_slice(&msg_hash)?, &sig), @@ -143,4 +143,4 @@ mod test { assert!(verify(c[1].as_bytes(), c[2], &PublicKey::from_str(c[3]).unwrap())) } } -} +} \ No newline at end of file diff --git a/lightning/src/util/mod.rs b/lightning/src/util/mod.rs index 7eace221779..e86885a83db 100644 --- a/lightning/src/util/mod.rs +++ b/lightning/src/util/mod.rs @@ -30,10 +30,6 @@ pub(crate) mod base32; pub(crate) mod atomic_counter; pub(crate) mod byte_utils; pub(crate) mod chacha20; -#[cfg(fuzzing)] -pub mod zbase32; -#[cfg(not(fuzzing))] -pub(crate) mod zbase32; #[cfg(not(fuzzing))] pub(crate) mod poly1305; pub(crate) mod chacha20poly1305rfc; diff --git a/lightning/src/util/zbase32.rs b/lightning/src/util/zbase32.rs deleted file mode 100644 index 8a7bf3516f1..00000000000 --- a/lightning/src/util/zbase32.rs +++ /dev/null @@ -1,144 +0,0 @@ -// This is a modification of base32 encoding to support the zbase32 alphabet. -// The original piece of software can be found at https://github.com/andreasots/base32 -// The original portions of this software are Copyright (c) 2015 The base32 Developers - -/* This file is licensed under either of - * Apache License, Version 2.0, (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or - * MIT license (LICENSE-MIT or http://opensource.org/licenses/MIT) - * at your option. -*/ - -use crate::prelude::*; - -const ALPHABET: &'static [u8] = b"ybndrfg8ejkmcpqxot1uwisza345h769"; - -/// Encodes some bytes as a zbase32 string -pub fn encode(data: &[u8]) -> String { - let mut ret = Vec::with_capacity((data.len() + 4) / 5 * 8); - - for chunk in data.chunks(5) { - let buf = { - let mut buf = [0u8; 5]; - for (i, &b) in chunk.iter().enumerate() { - buf[i] = b; - } - buf - }; - - ret.push(ALPHABET[((buf[0] & 0xF8) >> 3) as usize]); - ret.push(ALPHABET[(((buf[0] & 0x07) << 2) | ((buf[1] & 0xC0) >> 6)) as usize]); - ret.push(ALPHABET[((buf[1] & 0x3E) >> 1) as usize]); - ret.push(ALPHABET[(((buf[1] & 0x01) << 4) | ((buf[2] & 0xF0) >> 4)) as usize]); - ret.push(ALPHABET[(((buf[2] & 0x0F) << 1) | (buf[3] >> 7)) as usize]); - ret.push(ALPHABET[((buf[3] & 0x7C) >> 2) as usize]); - ret.push(ALPHABET[(((buf[3] & 0x03) << 3) | ((buf[4] & 0xE0) >> 5)) as usize]); - ret.push(ALPHABET[(buf[4] & 0x1F) as usize]); - } - - ret.truncate((data.len() * 8 + 4) / 5); - - // Check that our capacity calculation doesn't under-shoot in fuzzing - #[cfg(fuzzing)] - assert_eq!(ret.capacity(), (data.len() + 4) / 5 * 8); - - String::from_utf8(ret).unwrap() -} - -// ASCII 0-Z -const INV_ALPHABET: [i8; 43] = [ - -1, 18, -1, 25, 26, 27, 30, 29, 7, 31, -1, -1, -1, -1, -1, -1, -1, 24, 1, 12, 3, 8, 5, 6, 28, - 21, 9, 10, -1, 11, 2, 16, 13, 14, 4, 22, 17, 19, -1, 20, 15, 0, 23, -]; - -/// Decodes a zbase32 string to the original bytes, failing if the string was not encoded by a -/// proper zbase32 encoder. -pub fn decode(data: &str) -> Result, ()> { - if !data.is_ascii() { - return Err(()); - } - - let data = data.as_bytes(); - let output_length = data.len() * 5 / 8; - if data.len() > (output_length * 8 + 4) / 5 { - // If the string has more charachters than are required to encode the number of bytes - // decodable, treat the string as invalid. - return Err(()); - } - - let mut ret = Vec::with_capacity((data.len() + 7) / 8 * 5); - - for chunk in data.chunks(8) { - let buf = { - let mut buf = [0u8; 8]; - for (i, &c) in chunk.iter().enumerate() { - match INV_ALPHABET.get(c.to_ascii_uppercase().wrapping_sub(b'0') as usize) { - Some(&-1) | None => return Err(()), - Some(&value) => buf[i] = value as u8, - }; - } - buf - }; - ret.push((buf[0] << 3) | (buf[1] >> 2)); - ret.push((buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4)); - ret.push((buf[3] << 4) | (buf[4] >> 1)); - ret.push((buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3)); - ret.push((buf[6] << 5) | buf[7]); - } - for c in ret.drain(output_length..) { - if c != 0 { - // If the original string had any bits set at positions outside of the encoded data, - // treat the string as invalid. - return Err(()); - } - } - - // Check that our capacity calculation doesn't under-shoot in fuzzing - #[cfg(fuzzing)] - assert_eq!(ret.capacity(), (data.len() + 7) / 8 * 5); - - Ok(ret) -} - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_DATA: &[(&str, &[u8])] = &[ - ("", &[]), - ("yy", &[0x00]), - ("oy", &[0x80]), - ("tqrey", &[0x8b, 0x88, 0x80]), - ("6n9hq", &[0xf0, 0xbf, 0xc7]), - ("4t7ye", &[0xd4, 0x7a, 0x04]), - ("6im5sdy", &[0xf5, 0x57, 0xbb, 0x0c]), - ("ybndrfg8ejkmcpqxot1uwisza345h769", &[0x00, 0x44, 0x32, 0x14, 0xc7, 0x42, 0x54, 0xb6, - 0x35, 0xcf, 0x84, 0x65, 0x3a, 0x56, 0xd7, 0xc6, - 0x75, 0xbe, 0x77, 0xdf]) - ]; - - #[test] - fn test_encode() { - for &(zbase32, data) in TEST_DATA { - assert_eq!(encode(data), zbase32); - } - } - - #[test] - fn test_decode() { - for &(zbase32, data) in TEST_DATA { - assert_eq!(decode(zbase32).unwrap(), data); - } - } - - #[test] - fn test_decode_wrong() { - const WRONG_DATA: &[&str] = &["00", "l1", "?", "="]; - - for &data in WRONG_DATA { - match decode(data) { - Ok(_) => assert!(false, "Data shouldn't be decodable"), - Err(_) => assert!(true), - } - } - } -} From b40146457932e97482b63dc5b29671b77fe2b3db Mon Sep 17 00:00:00 2001 From: jbesraa Date: Mon, 4 Sep 2023 18:29:19 +0300 Subject: [PATCH 20/38] Fix indent in `message_signing.rs` --- lightning/src/util/message_signing.rs | 183 +++++++++++++------------- 1 file changed, 92 insertions(+), 91 deletions(-) diff --git a/lightning/src/util/message_signing.rs b/lightning/src/util/message_signing.rs index 4a159fc6327..477cbcbdd9a 100644 --- a/lightning/src/util/message_signing.rs +++ b/lightning/src/util/message_signing.rs @@ -29,118 +29,119 @@ use bitcoin::secp256k1::{Error, Message, PublicKey, Secp256k1, SecretKey}; static LN_MESSAGE_PREFIX: &[u8] = b"Lightning Signed Message:"; fn sigrec_encode(sig_rec: RecoverableSignature) -> Vec { - let (rid, rsig) = sig_rec.serialize_compact(); - let prefix = rid.to_i32() as u8 + 31; + let (rid, rsig) = sig_rec.serialize_compact(); + let prefix = rid.to_i32() as u8 + 31; - [&[prefix], &rsig[..]].concat() + [&[prefix], &rsig[..]].concat() } fn sigrec_decode(sig_rec: Vec) -> Result { - // Signature must be 64 + 1 bytes long (compact signature + recovery id) - if sig_rec.len() != 65 { - return Err(Error::InvalidSignature); - } - - let rsig = &sig_rec[1..]; - let rid = sig_rec[0] as i32 - 31; - - match RecoveryId::from_i32(rid) { - Ok(x) => RecoverableSignature::from_compact(rsig, x), - Err(e) => Err(e) - } + // Signature must be 64 + 1 bytes long (compact signature + recovery id) + if sig_rec.len() != 65 { + return Err(Error::InvalidSignature); + } + + let rsig = &sig_rec[1..]; + let rid = sig_rec[0] as i32 - 31; + + match RecoveryId::from_i32(rid) { + Ok(x) => RecoverableSignature::from_compact(rsig, x), + Err(e) => Err(e) + } } /// Creates a digital signature of a message given a SecretKey, like the node's secret. /// A receiver knowing the PublicKey (e.g. the node's id) and the message can be sure that the signature was generated by the caller. /// Signatures are EC recoverable, meaning that given the message and the signature the PublicKey of the signer can be extracted. pub fn sign(msg: &[u8], sk: &SecretKey) -> Result { - let secp_ctx = Secp256k1::signing_only(); - let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); + let secp_ctx = Secp256k1::signing_only(); + let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); - let sig = secp_ctx.sign_ecdsa_recoverable(&Message::from_slice(&msg_hash)?, sk); - Ok(base32::Alphabet::ZBase32.encode(&sigrec_encode(sig))) + let sig = secp_ctx.sign_ecdsa_recoverable(&Message::from_slice(&msg_hash)?, sk); + Ok(base32::Alphabet::ZBase32.encode(&sigrec_encode(sig))) } /// Recovers the PublicKey of the signer of the message given the message and the signature. pub fn recover_pk(msg: &[u8], sig: &str) -> Result { - let secp_ctx = Secp256k1::verification_only(); - let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); - - match base32::Alphabet::ZBase32.decode(&sig) { - Ok(sig_rec) => { - match sigrec_decode(sig_rec) { - Ok(sig) => secp_ctx.recover_ecdsa(&Message::from_slice(&msg_hash)?, &sig), - Err(e) => Err(e) - } - }, - Err(_) => Err(Error::InvalidSignature) - } + let secp_ctx = Secp256k1::verification_only(); + let msg_hash = sha256d::Hash::hash(&[LN_MESSAGE_PREFIX, msg].concat()); + + match base32::Alphabet::ZBase32.decode(&sig) { + Ok(sig_rec) => { + match sigrec_decode(sig_rec) { + Ok(sig) => secp_ctx.recover_ecdsa(&Message::from_slice(&msg_hash)?, &sig), + Err(e) => Err(e) + } + }, + Err(_) => Err(Error::InvalidSignature) + } } /// Verifies a message was signed by a PrivateKey that derives to a given PublicKey, given a message, a signature, /// and the PublicKey. pub fn verify(msg: &[u8], sig: &str, pk: &PublicKey) -> bool { - match recover_pk(msg, sig) { - Ok(x) => x == *pk, - Err(_) => false - } + match recover_pk(msg, sig) { + Ok(x) => x == *pk, + Err(_) => false + } } #[cfg(test)] mod test { - use core::str::FromStr; - use crate::util::message_signing::{sign, recover_pk, verify}; - use bitcoin::secp256k1::ONE_KEY; - use bitcoin::secp256k1::{PublicKey, Secp256k1}; - - #[test] - fn test_sign() { - let message = "test message"; - let zbase32_sig = sign(message.as_bytes(), &ONE_KEY); - - assert_eq!(zbase32_sig.unwrap(), "d9tibmnic9t5y41hg7hkakdcra94akas9ku3rmmj4ag9mritc8ok4p5qzefs78c9pqfhpuftqqzhydbdwfg7u6w6wdxcqpqn4sj4e73e") - } - - #[test] - fn test_recover_pk() { - let message = "test message"; - let sig = "d9tibmnic9t5y41hg7hkakdcra94akas9ku3rmmj4ag9mritc8ok4p5qzefs78c9pqfhpuftqqzhydbdwfg7u6w6wdxcqpqn4sj4e73e"; - let pk = recover_pk(message.as_bytes(), sig); - - assert_eq!(pk.unwrap(), PublicKey::from_secret_key(&Secp256k1::signing_only(), &ONE_KEY)) - } - - #[test] - fn test_verify() { - let message = "another message"; - let sig = sign(message.as_bytes(), &ONE_KEY).unwrap(); - let pk = PublicKey::from_secret_key(&Secp256k1::signing_only(), &ONE_KEY); - - assert!(verify(message.as_bytes(), &sig, &pk)) - } - - #[test] - fn test_verify_ground_truth_ish() { - // There are no standard tests vectors for Sign/Verify, using the same tests vectors as c-lightning to see if they are compatible. - // Taken from https://github.com/ElementsProject/lightning/blob/1275af6fbb02460c8eb2f00990bb0ef9179ce8f3/tests/test_misc.py#L1925-L1938 - - let corpus = [ - ["@bitconner", - "is this compatible?", - "rbgfioj114mh48d8egqx8o9qxqw4fmhe8jbeeabdioxnjk8z3t1ma1hu1fiswpakgucwwzwo6ofycffbsqusqdimugbh41n1g698hr9t", - "02b80cabdf82638aac86948e4c06e82064f547768dcef977677b9ea931ea75bab5"], - ["@duck1123", - "hi", - "rnrphcjswusbacjnmmmrynh9pqip7sy5cx695h6mfu64iac6qmcmsd8xnsyczwmpqp9shqkth3h4jmkgyqu5z47jfn1q7gpxtaqpx4xg", - "02de60d194e1ca5947b59fe8e2efd6aadeabfb67f2e89e13ae1a799c1e08e4a43b"], - ["@jochemin", - "hi", - "ry8bbsopmduhxy3dr5d9ekfeabdpimfx95kagdem7914wtca79jwamtbw4rxh69hg7n6x9ty8cqk33knbxaqftgxsfsaeprxkn1k48p3", - "022b8ece90ee891cbcdac0c1cc6af46b73c47212d8defbce80265ac81a6b794931"], - ]; - - for c in &corpus { - assert!(verify(c[1].as_bytes(), c[2], &PublicKey::from_str(c[3]).unwrap())) - } - } -} \ No newline at end of file + use core::str::FromStr; + use crate::util::message_signing::{sign, recover_pk, verify}; + use bitcoin::secp256k1::ONE_KEY; + use bitcoin::secp256k1::{PublicKey, Secp256k1}; + + #[test] + fn test_sign() { + let message = "test message"; + let zbase32_sig = sign(message.as_bytes(), &ONE_KEY); + + assert_eq!(zbase32_sig.unwrap(), "d9tibmnic9t5y41hg7hkakdcra94akas9ku3rmmj4ag9mritc8ok4p5qzefs78c9pqfhpuftqqzhydbdwfg7u6w6wdxcqpqn4sj4e73e") + } + + #[test] + fn test_recover_pk() { + let message = "test message"; + let sig = "d9tibmnic9t5y41hg7hkakdcra94akas9ku3rmmj4ag9mritc8ok4p5qzefs78c9pqfhpuftqqzhydbdwfg7u6w6wdxcqpqn4sj4e73e"; + let pk = recover_pk(message.as_bytes(), sig); + + assert_eq!(pk.unwrap(), PublicKey::from_secret_key(&Secp256k1::signing_only(), &ONE_KEY)) + } + + #[test] + fn test_verify() { + let message = "another message"; + let sig = sign(message.as_bytes(), &ONE_KEY).unwrap(); + let pk = PublicKey::from_secret_key(&Secp256k1::signing_only(), &ONE_KEY); + + assert!(verify(message.as_bytes(), &sig, &pk)) + } + + #[test] + fn test_verify_ground_truth_ish() { + // There are no standard tests vectors for Sign/Verify, using the same tests vectors as c-lightning to see if they are compatible. + // Taken from https://github.com/ElementsProject/lightning/blob/1275af6fbb02460c8eb2f00990bb0ef9179ce8f3/tests/test_misc.py#L1925-L1938 + + let corpus = [ + ["@bitconner", + "is this compatible?", + "rbgfioj114mh48d8egqx8o9qxqw4fmhe8jbeeabdioxnjk8z3t1ma1hu1fiswpakgucwwzwo6ofycffbsqusqdimugbh41n1g698hr9t", + "02b80cabdf82638aac86948e4c06e82064f547768dcef977677b9ea931ea75bab5"], + ["@duck1123", + "hi", + "rnrphcjswusbacjnmmmrynh9pqip7sy5cx695h6mfu64iac6qmcmsd8xnsyczwmpqp9shqkth3h4jmkgyqu5z47jfn1q7gpxtaqpx4xg", + "02de60d194e1ca5947b59fe8e2efd6aadeabfb67f2e89e13ae1a799c1e08e4a43b"], + ["@jochemin", + "hi", + "ry8bbsopmduhxy3dr5d9ekfeabdpimfx95kagdem7914wtca79jwamtbw4rxh69hg7n6x9ty8cqk33knbxaqftgxsfsaeprxkn1k48p3", + "022b8ece90ee891cbcdac0c1cc6af46b73c47212d8defbce80265ac81a6b794931"], + ]; + + for c in &corpus { + assert!(verify(c[1].as_bytes(), c[2], &PublicKey::from_str(c[3]).unwrap())) + } + } +} + From 4d31af3c0e606e1f995f37cb058506c37b254c7a Mon Sep 17 00:00:00 2001 From: jbesraa Date: Fri, 25 Aug 2023 05:57:56 +0300 Subject: [PATCH 21/38] Implement `from_str` trait for `NetAddress` - Add fuzz test for `NetAddress` `from_str` function --- fuzz/src/bin/fromstr_to_netaddress_target.rs | 113 +++++++++++++ fuzz/src/bin/gen_target.sh | 1 + fuzz/src/fromstr_to_netaddress.rs | 31 ++++ fuzz/src/lib.rs | 1 + fuzz/targets.h | 1 + lightning/src/ln/msgs.rs | 168 +++++++++++++++++-- 6 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 fuzz/src/bin/fromstr_to_netaddress_target.rs create mode 100644 fuzz/src/fromstr_to_netaddress.rs diff --git a/fuzz/src/bin/fromstr_to_netaddress_target.rs b/fuzz/src/bin/fromstr_to_netaddress_target.rs new file mode 100644 index 00000000000..29c984e6016 --- /dev/null +++ b/fuzz/src/bin/fromstr_to_netaddress_target.rs @@ -0,0 +1,113 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +// This file is auto-generated by gen_target.sh based on target_template.txt +// To modify it, modify target_template.txt and run gen_target.sh instead. + +#![cfg_attr(feature = "libfuzzer_fuzz", no_main)] + +#[cfg(not(fuzzing))] +compile_error!("Fuzz targets need cfg=fuzzing"); + +extern crate lightning_fuzz; +use lightning_fuzz::fromstr_to_netaddress::*; + +#[cfg(feature = "afl")] +#[macro_use] extern crate afl; +#[cfg(feature = "afl")] +fn main() { + fuzz!(|data| { + fromstr_to_netaddress_run(data.as_ptr(), data.len()); + }); +} + +#[cfg(feature = "honggfuzz")] +#[macro_use] extern crate honggfuzz; +#[cfg(feature = "honggfuzz")] +fn main() { + loop { + fuzz!(|data| { + fromstr_to_netaddress_run(data.as_ptr(), data.len()); + }); + } +} + +#[cfg(feature = "libfuzzer_fuzz")] +#[macro_use] extern crate libfuzzer_sys; +#[cfg(feature = "libfuzzer_fuzz")] +fuzz_target!(|data: &[u8]| { + fromstr_to_netaddress_run(data.as_ptr(), data.len()); +}); + +#[cfg(feature = "stdin_fuzz")] +fn main() { + use std::io::Read; + + let mut data = Vec::with_capacity(8192); + std::io::stdin().read_to_end(&mut data).unwrap(); + fromstr_to_netaddress_run(data.as_ptr(), data.len()); +} + +#[test] +fn run_test_cases() { + use std::fs; + use std::io::Read; + use lightning_fuzz::utils::test_logger::StringBuffer; + + use std::sync::{atomic, Arc}; + { + let data: Vec = vec![0]; + fromstr_to_netaddress_run(data.as_ptr(), data.len()); + } + let mut threads = Vec::new(); + let threads_running = Arc::new(atomic::AtomicUsize::new(0)); + if let Ok(tests) = fs::read_dir("test_cases/fromstr_to_netaddress") { + for test in tests { + let mut data: Vec = Vec::new(); + let path = test.unwrap().path(); + fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap(); + threads_running.fetch_add(1, atomic::Ordering::AcqRel); + + let thread_count_ref = Arc::clone(&threads_running); + let main_thread_ref = std::thread::current(); + threads.push((path.file_name().unwrap().to_str().unwrap().to_string(), + std::thread::spawn(move || { + let string_logger = StringBuffer::new(); + + let panic_logger = string_logger.clone(); + let res = if ::std::panic::catch_unwind(move || { + fromstr_to_netaddress_test(&data, panic_logger); + }).is_err() { + Some(string_logger.into_string()) + } else { None }; + thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel); + main_thread_ref.unpark(); + res + }) + )); + while threads_running.load(atomic::Ordering::Acquire) > 32 { + std::thread::park(); + } + } + } + let mut failed_outputs = Vec::new(); + for (test, thread) in threads.drain(..) { + if let Some(output) = thread.join().unwrap() { + println!("\nOutput of {}:\n{}\n", test, output); + failed_outputs.push(test); + } + } + if !failed_outputs.is_empty() { + println!("Test cases which failed: "); + for case in failed_outputs { + println!("{}", case); + } + panic!(); + } +} diff --git a/fuzz/src/bin/gen_target.sh b/fuzz/src/bin/gen_target.sh index 676bfb82156..2fa7debdf46 100755 --- a/fuzz/src/bin/gen_target.sh +++ b/fuzz/src/bin/gen_target.sh @@ -22,6 +22,7 @@ GEN_TEST zbase32 GEN_TEST indexedmap GEN_TEST onion_hop_data GEN_TEST base32 +GEN_TEST fromstr_to_netaddress GEN_TEST msg_accept_channel msg_targets:: GEN_TEST msg_announcement_signatures msg_targets:: diff --git a/fuzz/src/fromstr_to_netaddress.rs b/fuzz/src/fromstr_to_netaddress.rs new file mode 100644 index 00000000000..19984112369 --- /dev/null +++ b/fuzz/src/fromstr_to_netaddress.rs @@ -0,0 +1,31 @@ +// This file is Copyright its original authors, visible in version control +// history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +use lightning::ln::msgs::NetAddress; +use core::str::FromStr; + +use crate::utils::test_logger; + +#[inline] +pub fn do_test(data: &[u8]) { + if let Ok(s) = std::str::from_utf8(data) { + let _ = NetAddress::from_str(s); + } + +} + +pub fn fromstr_to_netaddress_test(data: &[u8], _out: Out) { + do_test(data); +} + +#[no_mangle] +pub extern "C" fn fromstr_to_netaddress_run(data: *const u8, datalen: usize) { + do_test(unsafe { std::slice::from_raw_parts(data, datalen) }); +} + diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 607924eff95..5b5cd69cf96 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -30,5 +30,6 @@ pub mod router; pub mod zbase32; pub mod onion_hop_data; pub mod base32; +pub mod fromstr_to_netaddress; pub mod msg_targets; diff --git a/fuzz/targets.h b/fuzz/targets.h index a17231c6d6d..cad0ac4d822 100644 --- a/fuzz/targets.h +++ b/fuzz/targets.h @@ -15,6 +15,7 @@ void zbase32_run(const unsigned char* data, size_t data_len); void indexedmap_run(const unsigned char* data, size_t data_len); void onion_hop_data_run(const unsigned char* data, size_t data_len); void base32_run(const unsigned char* data, size_t data_len); +void fromstr_to_netaddress_run(const unsigned char* data, size_t data_len); void msg_accept_channel_run(const unsigned char* data, size_t data_len); void msg_announcement_signatures_run(const unsigned char* data, size_t data_len); void msg_channel_reestablish_run(const unsigned char* data, size_t data_len); diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index e8a226932b4..d12dafb65af 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -37,14 +37,17 @@ use crate::ln::onion_utils; use crate::onion_message; use crate::prelude::*; +use core::convert::TryFrom; use core::fmt; use core::fmt::Debug; +use core::str::FromStr; use crate::io::{self, Read}; use crate::io_extras::read_to_end; use crate::events::{MessageSendEventsProvider, OnionMessageProvider}; use crate::util::logger; use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, WithoutLength, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname, TransactionU16LenLimited, BigSize}; +use crate::util::base32; use crate::routing::gossip::{NodeAlias, NodeId}; @@ -899,6 +902,104 @@ impl Readable for NetAddress { } } +/// [`NetAddress`] error variants +#[derive(Debug, Eq, PartialEq, Clone)] +pub enum NetAddressParseError { + /// Socket address (IPv4/IPv6) parsing error + SocketAddrParse, + /// Invalid input format + InvalidInput, + /// Invalid port + InvalidPort, + /// Invalid onion v3 address + InvalidOnionV3, +} + +impl fmt::Display for NetAddressParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + NetAddressParseError::SocketAddrParse => write!(f, "Socket address (IPv4/IPv6) parsing error"), + NetAddressParseError::InvalidInput => write!(f, "Invalid input format. \ + Expected: \":\", \"[]:\", \".onion:\" or \":\""), + NetAddressParseError::InvalidPort => write!(f, "Invalid port"), + NetAddressParseError::InvalidOnionV3 => write!(f, "Invalid onion v3 address"), + } + } +} + +#[cfg(feature = "std")] +impl From for NetAddress { + fn from(addr: std::net::SocketAddrV4) -> Self { + NetAddress::IPv4 { addr: addr.ip().octets(), port: addr.port() } + } +} + +#[cfg(feature = "std")] +impl From for NetAddress { + fn from(addr: std::net::SocketAddrV6) -> Self { + NetAddress::IPv6 { addr: addr.ip().octets(), port: addr.port() } + } +} + +#[cfg(feature = "std")] +impl From for NetAddress { + fn from(addr: std::net::SocketAddr) -> Self { + match addr { + std::net::SocketAddr::V4(addr) => addr.into(), + std::net::SocketAddr::V6(addr) => addr.into(), + } + } +} + +fn parse_onion_address(host: &str, port: u16) -> Result { + if host.ends_with(".onion") { + let domain = &host[..host.len() - ".onion".len()]; + if domain.len() != 56 { + return Err(NetAddressParseError::InvalidOnionV3); + } + let onion = base32::Alphabet::RFC4648 { padding: false }.decode(&domain).map_err(|_| NetAddressParseError::InvalidOnionV3)?; + if onion.len() != 35 { + return Err(NetAddressParseError::InvalidOnionV3); + } + let version = onion[0]; + let first_checksum_flag = onion[1]; + let second_checksum_flag = onion[2]; + let mut ed25519_pubkey = [0; 32]; + ed25519_pubkey.copy_from_slice(&onion[3..35]); + let checksum = u16::from_be_bytes([first_checksum_flag, second_checksum_flag]); + return Ok(NetAddress::OnionV3 { ed25519_pubkey, checksum, version, port }); + + } else { + return Err(NetAddressParseError::InvalidInput); + } +} + +#[cfg(feature = "std")] +impl FromStr for NetAddress { + type Err = NetAddressParseError; + + fn from_str(s: &str) -> Result { + match std::net::SocketAddr::from_str(s) { + Ok(addr) => Ok(addr.into()), + Err(_) => { + let trimmed_input = match s.rfind(":") { + Some(pos) => pos, + None => return Err(NetAddressParseError::InvalidInput), + }; + let host = &s[..trimmed_input]; + let port: u16 = s[trimmed_input + 1..].parse().map_err(|_| NetAddressParseError::InvalidPort)?; + if host.ends_with(".onion") { + return parse_onion_address(host, port); + }; + if let Ok(hostname) = Hostname::try_from(s[..trimmed_input].to_string()) { + return Ok(NetAddress::Hostname { hostname, port }); + }; + return Err(NetAddressParseError::SocketAddrParse) + }, + } + } +} + /// Represents the set of gossip messages that require a signature from a node's identity key. pub enum UnsignedGossipMessage<'a> { /// An unsigned channel announcement. @@ -2471,6 +2572,7 @@ impl_writeable_msg!(GossipTimestampFilter, { #[cfg(test)] mod tests { + use std::convert::TryFrom; use bitcoin::blockdata::constants::ChainHash; use bitcoin::{Transaction, PackedLockTime, TxIn, Script, Sequence, Witness, TxOut}; use hex; @@ -2478,6 +2580,7 @@ mod tests { use crate::ln::ChannelId; use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures}; use crate::ln::msgs::{self, FinalOnionHopData, OnionErrorPacket}; + use crate::ln::msgs::NetAddress; use crate::routing::gossip::{NodeAlias, NodeId}; use crate::util::ser::{Writeable, Readable, Hostname, TransactionU16LenLimited}; @@ -2493,11 +2596,13 @@ mod tests { use crate::io::{self, Cursor}; use crate::prelude::*; - use core::convert::TryFrom; use core::str::FromStr; - use crate::chain::transaction::OutPoint; + #[cfg(feature = "std")] + use std::net::{Ipv4Addr, Ipv6Addr}; + use crate::ln::msgs::NetAddressParseError; + #[test] fn encoding_channel_reestablish() { let public_key = { @@ -2663,24 +2768,24 @@ mod tests { }; let mut addresses = Vec::new(); if ipv4 { - addresses.push(msgs::NetAddress::IPv4 { + addresses.push(NetAddress::IPv4 { addr: [255, 254, 253, 252], port: 9735 }); } if ipv6 { - addresses.push(msgs::NetAddress::IPv6 { + addresses.push(NetAddress::IPv6 { addr: [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], port: 9735 }); } if onionv2 { - addresses.push(msgs::NetAddress::OnionV2( + addresses.push(NetAddress::OnionV2( [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 38, 7] )); } if onionv3 { - addresses.push(msgs::NetAddress::OnionV3 { + addresses.push(NetAddress::OnionV3 { ed25519_pubkey: [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224], checksum: 32, version: 16, @@ -2688,7 +2793,7 @@ mod tests { }); } if hostname { - addresses.push(msgs::NetAddress::Hostname { + addresses.push(NetAddress::Hostname { hostname: Hostname::try_from(String::from("host")).unwrap(), port: 9735, }); @@ -3296,10 +3401,10 @@ mod tests { let shutdown = msgs::Shutdown { channel_id: ChannelId::from_bytes([2; 32]), scriptpubkey: - if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey() } + if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey() } else if script_type == 2 { Address::p2sh(&script, Network::Testnet).unwrap().script_pubkey() } else if script_type == 3 { Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).unwrap().script_pubkey() } - else { Address::p2wsh(&script, Network::Testnet).script_pubkey() }, + else { Address::p2wsh(&script, Network::Testnet).script_pubkey() }, }; let encoded_value = shutdown.encode(); let mut target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202").unwrap(); @@ -3504,7 +3609,7 @@ mod tests { }.encode(), hex::decode("00000000014001010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202").unwrap()); let init_msg = msgs::Init { features: InitFeatures::from_le_bytes(vec![]), networks: Some(vec![mainnet_hash]), - remote_network_address: Some(msgs::NetAddress::IPv4 { + remote_network_address: Some(NetAddress::IPv4 { addr: [127, 0, 0, 1], port: 1000, }), @@ -3869,4 +3974,47 @@ mod tests { } Ok(encoded_payload) } + + #[test] + #[cfg(feature = "std")] + fn test_net_address_from_str() { + assert_eq!(NetAddress::IPv4 { + addr: Ipv4Addr::new(127, 0, 0, 1).octets(), + port: 1234, + }, NetAddress::from_str("127.0.0.1:1234").unwrap()); + + assert_eq!(NetAddress::IPv6 { + addr: Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).octets(), + port: 1234, + }, NetAddress::from_str("[0:0:0:0:0:0:0:1]:1234").unwrap()); + assert_eq!( + NetAddress::Hostname { + hostname: Hostname::try_from("lightning-node.mydomain.com".to_string()).unwrap(), + port: 1234, + }, NetAddress::from_str("lightning-node.mydomain.com:1234").unwrap()); + assert_eq!( + NetAddress::Hostname { + hostname: Hostname::try_from("example.com".to_string()).unwrap(), + port: 1234, + }, NetAddress::from_str("example.com:1234").unwrap()); + assert_eq!(NetAddress::OnionV3 { + ed25519_pubkey: [37, 24, 75, 5, 25, 73, 117, 194, 139, 102, 182, 107, 4, 105, 247, 246, 85, + 111, 177, 172, 49, 137, 167, 155, 64, 221, 163, 47, 31, 33, 71, 3], + checksum: 48326, + version: 121, + port: 1234 + }, NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:1234").unwrap()); + assert_eq!(Err(NetAddressParseError::InvalidOnionV3), NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6.onion:1234")); + assert_eq!(Err(NetAddressParseError::InvalidInput), NetAddress::from_str("127.0.0.1@1234")); + assert_eq!(Err(NetAddressParseError::InvalidInput), "".parse::()); + assert!(NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:9735:94").is_err()); + assert!(NetAddress::from_str("wrong$%#.com:1234").is_err()); + assert_eq!(Err(NetAddressParseError::InvalidPort), NetAddress::from_str("example.com:wrong")); + assert!("localhost".parse::().is_err()); + assert!("localhost:invalid-port".parse::().is_err()); + assert!( "invalid-onion-v3-hostname.onion:8080".parse::().is_err()); + assert!("b32.example.onion:invalid-port".parse::().is_err()); + assert!("invalid-address".parse::().is_err()); + assert!(NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:1234").is_err()); + } } From ef142926a6fd238e7073704348508caf6660aee4 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Tue, 1 Aug 2023 10:24:53 +0200 Subject: [PATCH 22/38] Add `KVStore` interface trait We upstream the `KVStore` interface trait from LDK Node, which will replace `KVStorePersister` in the coming commits. Besides persistence, `KVStore` implementations will also offer to `list` keys present in a given `namespace` and `read` the stored values. --- lightning/src/util/persist.rs | 92 ++++++++++++++++++++++++++++++++++- pending_changelog/kvstore.txt | 3 ++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 pending_changelog/kvstore.txt diff --git a/lightning/src/util/persist.rs b/lightning/src/util/persist.rs index 435ef30d331..35d19eea46b 100644 --- a/lightning/src/util/persist.rs +++ b/lightning/src/util/persist.rs @@ -4,13 +4,14 @@ // You may not use this file except in accordance with one or both of these // licenses. -//! This module contains a simple key-value store trait KVStorePersister that +//! This module contains a simple key-value store trait [`KVStore`] that //! allows one to implement the persistence for [`ChannelManager`], [`NetworkGraph`], //! and [`ChannelMonitor`] all in one place. use core::ops::Deref; use bitcoin::hashes::hex::ToHex; use crate::io; +use crate::prelude::{Vec, String}; use crate::routing::scoring::WriteableScore; use crate::chain; @@ -22,7 +23,94 @@ use crate::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}; use crate::ln::channelmanager::ChannelManager; use crate::routing::router::Router; use crate::routing::gossip::NetworkGraph; -use super::{logger::Logger, ser::Writeable}; +use crate::util::logger::Logger; +use crate::util::ser::Writeable; + +/// The alphabet of characters allowed for namespaces and keys. +pub const KVSTORE_NAMESPACE_KEY_ALPHABET: &str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"; + +/// The maximum number of characters namespaces and keys may have. +pub const KVSTORE_NAMESPACE_KEY_MAX_LEN: usize = 120; + +/// The namespace under which the [`ChannelManager`] will be persisted. +pub const CHANNEL_MANAGER_PERSISTENCE_NAMESPACE: &str = ""; +/// The sub-namespace under which the [`ChannelManager`] will be persisted. +pub const CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE: &str = ""; +/// The key under which the [`ChannelManager`] will be persisted. +pub const CHANNEL_MANAGER_PERSISTENCE_KEY: &str = "manager"; + +/// The namespace under which [`ChannelMonitor`]s will be persisted. +pub const CHANNEL_MONITOR_PERSISTENCE_NAMESPACE: &str = "monitors"; +/// The sub-namespace under which [`ChannelMonitor`]s will be persisted. +pub const CHANNEL_MONITOR_PERSISTENCE_SUB_NAMESPACE: &str = ""; + +/// The namespace under which the [`NetworkGraph`] will be persisted. +pub const NETWORK_GRAPH_PERSISTENCE_NAMESPACE: &str = ""; +/// The sub-namespace under which the [`NetworkGraph`] will be persisted. +pub const NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE: &str = ""; +/// The key under which the [`NetworkGraph`] will be persisted. +pub const NETWORK_GRAPH_PERSISTENCE_KEY: &str = "network_graph"; + +/// The namespace under which the [`WriteableScore`] will be persisted. +pub const SCORER_PERSISTENCE_NAMESPACE: &str = ""; +/// The sub-namespace under which the [`WriteableScore`] will be persisted. +pub const SCORER_PERSISTENCE_SUB_NAMESPACE: &str = ""; +/// The key under which the [`WriteableScore`] will be persisted. +pub const SCORER_PERSISTENCE_KEY: &str = "scorer"; + +/// Provides an interface that allows storage and retrieval of persisted values that are associated +/// with given keys. +/// +/// In order to avoid collisions the key space is segmented based on the given `namespace`s and +/// `sub_namespace`s. Implementations of this trait are free to handle them in different ways, as +/// long as per-namespace key uniqueness is asserted. +/// +/// Keys and namespaces are required to be valid ASCII strings in the range of +/// [`KVSTORE_NAMESPACE_KEY_ALPHABET`] and no longer than [`KVSTORE_NAMESPACE_KEY_MAX_LEN`]. Empty +/// namespaces and sub-namespaces (`""`) are assumed to be a valid, however, if `namespace` is +/// empty, `sub_namespace` is required to be empty, too. This means that concerns should always be +/// separated by namespace first, before sub-namespaces are used. While the number of namespaces +/// will be relatively small and is determined at compile time, there may be many sub-namespaces +/// per namespace. Note that per-namespace uniqueness needs to also hold for keys *and* +/// namespaces/sub-namespaces in any given namespace/sub-namespace, i.e., conflicts between keys +/// and equally named namespaces/sub-namespaces must be avoided. +/// +/// **Note:** Users migrating custom persistence backends from the pre-v0.0.117 `KVStorePersister` +/// interface can use a concatenation of `[{namespace}/[{sub_namespace}/]]{key}` to recover a `key` compatible with the +/// data model previously assumed by `KVStorePersister::persist`. +pub trait KVStore { + /// Returns the data stored for the given `namespace`, `sub_namespace`, and `key`. + /// + /// Returns an [`ErrorKind::NotFound`] if the given `key` could not be found in the given + /// `namespace` and `sub_namespace`. + /// + /// [`ErrorKind::NotFound`]: io::ErrorKind::NotFound + fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result>; + /// Persists the given data under the given `key`. + /// + /// Will create the given `namespace` and `sub_namespace` if not already present in the store. + fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()>; + /// Removes any data that had previously been persisted under the given `key`. + /// + /// If the `lazy` flag is set to `true`, the backend implementation might choose to lazily + /// remove the given `key` at some point in time after the method returns, e.g., as part of an + /// eventual batch deletion of multiple keys. As a consequence, subsequent calls to + /// [`KVStore::list`] might include the removed key until the changes are actually persisted. + /// + /// Note that while setting the `lazy` flag reduces the I/O burden of multiple subsequent + /// `remove` calls, it also influences the atomicity guarantees as lazy `remove`s could + /// potentially get lost on crash after the method returns. Therefore, this flag should only be + /// set for `remove` operations that can be safely replayed at a later time. + /// + /// Returns successfully if no data will be stored for the given `namespace`, `sub_namespace`, and + /// `key`, independently of whether it was present before its invokation or not. + fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> io::Result<()>; + /// Returns a list of keys that are stored under the given `sub_namespace` in `namespace`. + /// + /// Returns the keys in arbitrary order, so users requiring a particular order need to sort the + /// returned keys. Returns an empty list if `namespace` or `sub_namespace` is unknown. + fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result>; +} /// Trait for a key-value store for persisting some writeable object at some key /// Implementing `KVStorePersister` provides auto-implementations for [`Persister`] diff --git a/pending_changelog/kvstore.txt b/pending_changelog/kvstore.txt new file mode 100644 index 00000000000..d96fd69371b --- /dev/null +++ b/pending_changelog/kvstore.txt @@ -0,0 +1,3 @@ +## Backwards Compatibility + +* Users migrating custom persistence backends from the pre-v0.0.117 `KVStorePersister` interface can use a concatenation of `[{namespace}/[{sub_namespace}/]]{key}` to recover a `key` compatible with the data model previously assumed by `KVStorePersister::persist`. From e82f4273492741f34620625462e7823c148b3e8c Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 21 Aug 2023 13:13:56 +0200 Subject: [PATCH 23/38] Update `lightning-persister` crate --- lightning-persister/Cargo.toml | 4 ++-- lightning-persister/src/lib.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index 35bddc07746..a6775a18cff 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -3,9 +3,9 @@ name = "lightning-persister" version = "0.0.116" authors = ["Valentine Wallace", "Matt Corallo"] license = "MIT OR Apache-2.0" -repository = "https://github.com/lightningdevkit/rust-lightning/" +repository = "https://github.com/lightningdevkit/rust-lightning" description = """ -Utilities to manage Rust-Lightning channel data persistence and retrieval. +Utilities for LDK data persistence and retrieval. """ edition = "2018" diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index b34fe895b47..dc205feba4e 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -1,6 +1,6 @@ -//! Utilities that handle persisting Rust-Lightning data to disk via standard filesystem APIs. - -// Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. +//! Provides utilities for LDK data persistence and retrieval. +// +// TODO: Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings. #![deny(broken_intra_doc_links)] #![deny(private_intra_doc_links)] From 5591d3484ce53c828aee493975ff699f15e0b54b Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 4 Aug 2023 10:09:55 +0200 Subject: [PATCH 24/38] Add `test_utils` We add a utility function needed by upcoming `KVStore` implementation tests. --- lightning-persister/src/lib.rs | 3 ++ lightning-persister/src/test_utils.rs | 50 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 lightning-persister/src/test_utils.rs diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index dc205feba4e..0a667e4216f 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -10,6 +10,9 @@ #[cfg(ldk_bench)] extern crate criterion; +#[cfg(test)] +mod test_utils; + mod util; extern crate lightning; diff --git a/lightning-persister/src/test_utils.rs b/lightning-persister/src/test_utils.rs new file mode 100644 index 00000000000..357a8b2ee8c --- /dev/null +++ b/lightning-persister/src/test_utils.rs @@ -0,0 +1,50 @@ +use lightning::util::persist::{KVStore, KVSTORE_NAMESPACE_KEY_MAX_LEN}; + +use std::panic::RefUnwindSafe; + +pub(crate) fn do_read_write_remove_list_persist(kv_store: &K) { + let data = [42u8; 32]; + + let namespace = "testspace"; + let sub_namespace = "testsubspace"; + let key = "testkey"; + + // Test the basic KVStore operations. + kv_store.write(namespace, sub_namespace, key, &data).unwrap(); + + // Test empty namespace/sub_namespace is allowed, but not empty namespace and non-empty + // sub-namespace, and not empty key. + kv_store.write("", "", key, &data).unwrap(); + let res = std::panic::catch_unwind(|| kv_store.write("", sub_namespace, key, &data)); + assert!(res.is_err()); + let res = std::panic::catch_unwind(|| kv_store.write(namespace, sub_namespace, "", &data)); + assert!(res.is_err()); + + let listed_keys = kv_store.list(namespace, sub_namespace).unwrap(); + assert_eq!(listed_keys.len(), 1); + assert_eq!(listed_keys[0], key); + + let read_data = kv_store.read(namespace, sub_namespace, key).unwrap(); + assert_eq!(data, &*read_data); + + kv_store.remove(namespace, sub_namespace, key, false).unwrap(); + + let listed_keys = kv_store.list(namespace, sub_namespace).unwrap(); + assert_eq!(listed_keys.len(), 0); + + // Ensure we have no issue operating with namespace/sub_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN + let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect(); + kv_store.write(&max_chars, &max_chars, &max_chars, &data).unwrap(); + + let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + assert_eq!(listed_keys.len(), 1); + assert_eq!(listed_keys[0], max_chars); + + let read_data = kv_store.read(&max_chars, &max_chars, &max_chars).unwrap(); + assert_eq!(data, &*read_data); + + kv_store.remove(&max_chars, &max_chars, &max_chars, false).unwrap(); + + let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); + assert_eq!(listed_keys.len(), 0); +} From e9321a52a9ceb0c6285144acfa04e787dfbe64a9 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Tue, 1 Aug 2023 10:46:51 +0200 Subject: [PATCH 25/38] Add `FilesystemStore` We upstream the `FilesystemStore` implementation, which is backwards compatible with `lightning-persister::FilesystemPersister`. --- lightning-persister/Cargo.toml | 1 + lightning-persister/src/fs_store.rs | 379 ++++++++++++++++++++++++++++ lightning-persister/src/lib.rs | 4 + lightning-persister/src/utils.rs | 59 +++++ 4 files changed, 443 insertions(+) create mode 100644 lightning-persister/src/fs_store.rs create mode 100644 lightning-persister/src/utils.rs diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index a6775a18cff..271f3b882b3 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -20,6 +20,7 @@ libc = "0.2" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["winbase"] } +windows-sys = { version = "0.48.0", default-features = false, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] } [target.'cfg(ldk_bench)'.dependencies] criterion = { version = "0.4", optional = true, default-features = false } diff --git a/lightning-persister/src/fs_store.rs b/lightning-persister/src/fs_store.rs new file mode 100644 index 00000000000..f74806d7d8e --- /dev/null +++ b/lightning-persister/src/fs_store.rs @@ -0,0 +1,379 @@ +//! Objects related to [`FilesystemStore`] live here. +use crate::utils::{check_namespace_key_validity, is_valid_kvstore_str}; + +use lightning::util::persist::KVStore; +use lightning::util::string::PrintableString; + +use std::collections::HashMap; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +#[cfg(target_os = "windows")] +use {std::ffi::OsStr, std::os::windows::ffi::OsStrExt}; + +#[cfg(target_os = "windows")] +macro_rules! call { + ($e: expr) => { + if $e != 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }; +} + +#[cfg(target_os = "windows")] +fn path_to_windows_str>(path: T) -> Vec { + path.as_ref().encode_wide().chain(Some(0)).collect() +} + +// The number of read/write/remove/list operations after which we clean up our `locks` HashMap. +const GC_LOCK_INTERVAL: usize = 25; + +/// A [`KVStore`] implementation that writes to and reads from the file system. +pub struct FilesystemStore { + data_dir: PathBuf, + tmp_file_counter: AtomicUsize, + gc_counter: AtomicUsize, + locks: Mutex>>>, +} + +impl FilesystemStore { + /// Constructs a new [`FilesystemStore`]. + pub fn new(data_dir: PathBuf) -> Self { + let locks = Mutex::new(HashMap::new()); + let tmp_file_counter = AtomicUsize::new(0); + let gc_counter = AtomicUsize::new(1); + Self { data_dir, tmp_file_counter, gc_counter, locks } + } + + /// Returns the data directory. + pub fn get_data_dir(&self) -> PathBuf { + self.data_dir.clone() + } + + fn garbage_collect_locks(&self) { + let gc_counter = self.gc_counter.fetch_add(1, Ordering::AcqRel); + + if gc_counter % GC_LOCK_INTERVAL == 0 { + // Take outer lock for the cleanup. + let mut outer_lock = self.locks.lock().unwrap(); + + // Garbage collect all lock entries that are not referenced anymore. + outer_lock.retain(|_, v| Arc::strong_count(&v) > 1); + } + } + + fn get_dest_dir_path(&self, namespace: &str, sub_namespace: &str) -> std::io::Result { + let mut dest_dir_path = { + #[cfg(target_os = "windows")] + { + let data_dir = self.data_dir.clone(); + fs::create_dir_all(data_dir.clone())?; + fs::canonicalize(data_dir)? + } + #[cfg(not(target_os = "windows"))] + { + self.data_dir.clone() + } + }; + + dest_dir_path.push(namespace); + if !sub_namespace.is_empty() { + dest_dir_path.push(sub_namespace); + } + + Ok(dest_dir_path) + } +} + +impl KVStore for FilesystemStore { + fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> std::io::Result> { + check_namespace_key_validity(namespace, sub_namespace, Some(key), "read")?; + + let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?; + dest_file_path.push(key); + + let mut buf = Vec::new(); + { + let inner_lock_ref = { + let mut outer_lock = self.locks.lock().unwrap(); + Arc::clone(&outer_lock.entry(dest_file_path.clone()).or_default()) + }; + let _guard = inner_lock_ref.read().unwrap(); + + let mut f = fs::File::open(dest_file_path)?; + f.read_to_end(&mut buf)?; + } + + self.garbage_collect_locks(); + + Ok(buf) + } + + fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> { + check_namespace_key_validity(namespace, sub_namespace, Some(key), "write")?; + + let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?; + dest_file_path.push(key); + + let parent_directory = dest_file_path + .parent() + .ok_or_else(|| { + let msg = + format!("Could not retrieve parent directory of {}.", dest_file_path.display()); + std::io::Error::new(std::io::ErrorKind::InvalidInput, msg) + })?; + fs::create_dir_all(&parent_directory)?; + + // Do a crazy dance with lots of fsync()s to be overly cautious here... + // We never want to end up in a state where we've lost the old data, or end up using the + // old data on power loss after we've returned. + // The way to atomically write a file on Unix platforms is: + // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) + let mut tmp_file_path = dest_file_path.clone(); + let tmp_file_ext = format!("{}.tmp", self.tmp_file_counter.fetch_add(1, Ordering::AcqRel)); + tmp_file_path.set_extension(tmp_file_ext); + + { + let mut tmp_file = fs::File::create(&tmp_file_path)?; + tmp_file.write_all(&buf)?; + tmp_file.sync_all()?; + } + + let res = { + let inner_lock_ref = { + let mut outer_lock = self.locks.lock().unwrap(); + Arc::clone(&outer_lock.entry(dest_file_path.clone()).or_default()) + }; + let _guard = inner_lock_ref.write().unwrap(); + + #[cfg(not(target_os = "windows"))] + { + fs::rename(&tmp_file_path, &dest_file_path)?; + let dir_file = fs::OpenOptions::new().read(true).open(&parent_directory)?; + dir_file.sync_all()?; + Ok(()) + } + + #[cfg(target_os = "windows")] + { + let res = if dest_file_path.exists() { + call!(unsafe { + windows_sys::Win32::Storage::FileSystem::ReplaceFileW( + path_to_windows_str(dest_file_path.clone()).as_ptr(), + path_to_windows_str(tmp_file_path).as_ptr(), + std::ptr::null(), + windows_sys::Win32::Storage::FileSystem::REPLACEFILE_IGNORE_MERGE_ERRORS, + std::ptr::null_mut() as *const core::ffi::c_void, + std::ptr::null_mut() as *const core::ffi::c_void, + ) + }) + } else { + call!(unsafe { + windows_sys::Win32::Storage::FileSystem::MoveFileExW( + path_to_windows_str(tmp_file_path).as_ptr(), + path_to_windows_str(dest_file_path.clone()).as_ptr(), + windows_sys::Win32::Storage::FileSystem::MOVEFILE_WRITE_THROUGH + | windows_sys::Win32::Storage::FileSystem::MOVEFILE_REPLACE_EXISTING, + ) + }) + }; + + match res { + Ok(()) => { + // We fsync the dest file in hopes this will also flush the metadata to disk. + let dest_file = fs::OpenOptions::new().read(true).write(true) + .open(&dest_file_path)?; + dest_file.sync_all()?; + Ok(()) + } + Err(e) => Err(e), + } + } + }; + + self.garbage_collect_locks(); + + res + } + + fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> { + check_namespace_key_validity(namespace, sub_namespace, Some(key), "remove")?; + + let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?; + dest_file_path.push(key); + + if !dest_file_path.is_file() { + return Ok(()); + } + + { + let inner_lock_ref = { + let mut outer_lock = self.locks.lock().unwrap(); + Arc::clone(&outer_lock.entry(dest_file_path.clone()).or_default()) + }; + let _guard = inner_lock_ref.write().unwrap(); + + if lazy { + // If we're lazy we just call remove and be done with it. + fs::remove_file(&dest_file_path)?; + } else { + // If we're not lazy we try our best to persist the updated metadata to ensure + // atomicity of this call. + #[cfg(not(target_os = "windows"))] + { + fs::remove_file(&dest_file_path)?; + + let parent_directory = dest_file_path.parent().ok_or_else(|| { + let msg = + format!("Could not retrieve parent directory of {}.", dest_file_path.display()); + std::io::Error::new(std::io::ErrorKind::InvalidInput, msg) + })?; + let dir_file = fs::OpenOptions::new().read(true).open(parent_directory)?; + // The above call to `fs::remove_file` corresponds to POSIX `unlink`, whose changes + // to the inode might get cached (and hence possibly lost on crash), depending on + // the target platform and file system. + // + // In order to assert we permanently removed the file in question we therefore + // call `fsync` on the parent directory on platforms that support it. + dir_file.sync_all()?; + } + + #[cfg(target_os = "windows")] + { + // Since Windows `DeleteFile` API is not persisted until the last open file handle + // is dropped, and there seemingly is no reliable way to flush the directory + // metadata, we here fall back to use a 'recycling bin' model, i.e., first move the + // file to be deleted to a temporary trash file and remove the latter file + // afterwards. + // + // This should be marginally better, as, according to the documentation, + // `MoveFileExW` APIs should offer stronger persistence guarantees, + // at least if `MOVEFILE_WRITE_THROUGH`/`MOVEFILE_REPLACE_EXISTING` is set. + // However, all this is partially based on assumptions and local experiments, as + // Windows API is horribly underdocumented. + let mut trash_file_path = dest_file_path.clone(); + let trash_file_ext = format!("{}.trash", + self.tmp_file_counter.fetch_add(1, Ordering::AcqRel)); + trash_file_path.set_extension(trash_file_ext); + + call!(unsafe { + windows_sys::Win32::Storage::FileSystem::MoveFileExW( + path_to_windows_str(dest_file_path).as_ptr(), + path_to_windows_str(trash_file_path.clone()).as_ptr(), + windows_sys::Win32::Storage::FileSystem::MOVEFILE_WRITE_THROUGH + | windows_sys::Win32::Storage::FileSystem::MOVEFILE_REPLACE_EXISTING, + ) + })?; + + { + // We fsync the trash file in hopes this will also flush the original's file + // metadata to disk. + let trash_file = fs::OpenOptions::new().read(true).write(true) + .open(&trash_file_path.clone())?; + trash_file.sync_all()?; + } + + // We're fine if this remove would fail as the trash file will be cleaned up in + // list eventually. + fs::remove_file(trash_file_path).ok(); + } + } + } + + self.garbage_collect_locks(); + + Ok(()) + } + + fn list(&self, namespace: &str, sub_namespace: &str) -> std::io::Result> { + check_namespace_key_validity(namespace, sub_namespace, None, "list")?; + + let prefixed_dest = self.get_dest_dir_path(namespace, sub_namespace)?; + let mut keys = Vec::new(); + + if !Path::new(&prefixed_dest).exists() { + return Ok(Vec::new()); + } + + for entry in fs::read_dir(&prefixed_dest)? { + let entry = entry?; + let p = entry.path(); + + if let Some(ext) = p.extension() { + #[cfg(target_os = "windows")] + { + // Clean up any trash files lying around. + if ext == "trash" { + fs::remove_file(p).ok(); + continue; + } + } + if ext == "tmp" { + continue; + } + } + + let metadata = p.metadata()?; + + // We allow the presence of directories in the empty namespace and just skip them. + if metadata.is_dir() { + continue; + } + + // If we otherwise don't find a file at the given path something went wrong. + if !metadata.is_file() { + debug_assert!(false, "Failed to list keys of {}/{}: file couldn't be accessed.", + PrintableString(namespace), PrintableString(sub_namespace)); + let msg = format!("Failed to list keys of {}/{}: file couldn't be accessed.", + PrintableString(namespace), PrintableString(sub_namespace)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + match p.strip_prefix(&prefixed_dest) { + Ok(stripped_path) => { + if let Some(relative_path) = stripped_path.to_str() { + if is_valid_kvstore_str(relative_path) { + keys.push(relative_path.to_string()) + } + } else { + debug_assert!(false, "Failed to list keys of {}/{}: file path is not valid UTF-8", + PrintableString(namespace), PrintableString(sub_namespace)); + let msg = format!("Failed to list keys of {}/{}: file path is not valid UTF-8", + PrintableString(namespace), PrintableString(sub_namespace)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + } + Err(e) => { + debug_assert!(false, "Failed to list keys of {}/{}: {}", + PrintableString(namespace), PrintableString(sub_namespace), e); + let msg = format!("Failed to list keys of {}/{}: {}", + PrintableString(namespace), PrintableString(sub_namespace), e); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + } + } + + self.garbage_collect_locks(); + + Ok(keys) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::do_read_write_remove_list_persist; + + #[test] + fn read_write_remove_list_persist() { + let mut temp_path = std::env::temp_dir(); + temp_path.push("test_read_write_remove_list_persist"); + let fs_store = FilesystemStore::new(temp_path); + do_read_write_remove_list_persist(&fs_store); + } +} diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index 0a667e4216f..932e4f41ad3 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -10,6 +10,10 @@ #[cfg(ldk_bench)] extern crate criterion; +pub mod fs_store; + +mod utils; + #[cfg(test)] mod test_utils; diff --git a/lightning-persister/src/utils.rs b/lightning-persister/src/utils.rs new file mode 100644 index 00000000000..54ec230de2d --- /dev/null +++ b/lightning-persister/src/utils.rs @@ -0,0 +1,59 @@ +use lightning::util::persist::{KVSTORE_NAMESPACE_KEY_ALPHABET, KVSTORE_NAMESPACE_KEY_MAX_LEN}; +use lightning::util::string::PrintableString; + + +pub(crate) fn is_valid_kvstore_str(key: &str) -> bool { + key.len() <= KVSTORE_NAMESPACE_KEY_MAX_LEN && key.chars().all(|c| KVSTORE_NAMESPACE_KEY_ALPHABET.contains(c)) +} + +pub(crate) fn check_namespace_key_validity(namespace: &str, sub_namespace: &str, key: Option<&str>, operation: &str) -> Result<(), std::io::Error> { + if let Some(key) = key { + if key.is_empty() { + debug_assert!(false, "Failed to {} {}/{}/{}: key may not be empty.", operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + let msg = format!("Failed to {} {}/{}/{}: key may not be empty.", operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + if namespace.is_empty() && !sub_namespace.is_empty() { + debug_assert!(false, + "Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.", + operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + let msg = format!( + "Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.", operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) || !is_valid_kvstore_str(key) { + debug_assert!(false, "Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.", + operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + let msg = format!("Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.", + operation, + PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + } else { + if namespace.is_empty() && !sub_namespace.is_empty() { + debug_assert!(false, + "Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.", + operation, PrintableString(namespace), PrintableString(sub_namespace)); + let msg = format!( + "Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.", + operation, PrintableString(namespace), PrintableString(sub_namespace)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) { + debug_assert!(false, "Failed to {} {}/{}: namespace and sub-namespace must be valid.", + operation, PrintableString(namespace), PrintableString(sub_namespace)); + let msg = format!("Failed to {} {}/{}: namespace and sub-namespace must be valid.", + operation, PrintableString(namespace), PrintableString(sub_namespace)); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + } + + Ok(()) +} From 485cc596d4035086cd424711a25d8e46f3b40f0e Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 4 Aug 2023 16:20:50 +0200 Subject: [PATCH 26/38] Add `read_channel_monitors` utility This replaces the `FilesystemPersister::read_channelmonitors` method, as we can now implement a single utility for all `KVStore`s. --- lightning/src/util/persist.rs | 55 +++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/lightning/src/util/persist.rs b/lightning/src/util/persist.rs index 35d19eea46b..5b122b7a570 100644 --- a/lightning/src/util/persist.rs +++ b/lightning/src/util/persist.rs @@ -9,7 +9,9 @@ //! and [`ChannelMonitor`] all in one place. use core::ops::Deref; -use bitcoin::hashes::hex::ToHex; +use bitcoin::hashes::hex::{FromHex, ToHex}; +use bitcoin::{BlockHash, Txid}; + use crate::io; use crate::prelude::{Vec, String}; use crate::routing::scoring::WriteableScore; @@ -24,7 +26,7 @@ use crate::ln::channelmanager::ChannelManager; use crate::routing::router::Router; use crate::routing::gossip::NetworkGraph; use crate::util::logger::Logger; -use crate::util::ser::Writeable; +use crate::util::ser::{ReadableArgs, Writeable}; /// The alphabet of characters allowed for namespaces and keys. pub const KVSTORE_NAMESPACE_KEY_ALPHABET: &str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"; @@ -190,3 +192,52 @@ impl Persist( + kv_store: K, entropy_source: ES, signer_provider: SP, +) -> io::Result::Signer>)>> +where + K::Target: KVStore, + ES::Target: EntropySource + Sized, + SP::Target: SignerProvider + Sized, +{ + let mut res = Vec::new(); + + for stored_key in kv_store.list( + CHANNEL_MONITOR_PERSISTENCE_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SUB_NAMESPACE)? + { + let txid = Txid::from_hex(stored_key.split_at(64).0).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid tx ID in stored key") + })?; + + let index: u16 = stored_key.split_at(65).1.parse().map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid tx index in stored key") + })?; + + match <(BlockHash, ChannelMonitor<::Signer>)>::read( + &mut io::Cursor::new( + kv_store.read(CHANNEL_MONITOR_PERSISTENCE_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SUB_NAMESPACE, &stored_key)?), + (&*entropy_source, &*signer_provider), + ) { + Ok((block_hash, channel_monitor)) => { + if channel_monitor.get_funding_txo().0.txid != txid + || channel_monitor.get_funding_txo().0.index != index + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ChannelMonitor was stored under the wrong key", + )); + } + res.push((block_hash, channel_monitor)); + } + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to deserialize ChannelMonitor" + )) + } + } + } + Ok(res) +} From a7ce0e3882a2165559160e26c3e562ef67873f0d Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Tue, 1 Aug 2023 13:37:46 +0200 Subject: [PATCH 27/38] Migrate to `KVStore`/`FilesystemStore` Firstly, we switch our BP over to use `FilesystemStore`, which also gives us test coverage and ensures the compatibility. Then, we remove the superseded `KVStorePersister` trait and the `FilesystemPersister` code. --- bench/benches/bench.rs | 1 - lightning-background-processor/src/lib.rs | 103 ++++--- lightning-persister/Cargo.toml | 2 - lightning-persister/src/lib.rs | 338 ---------------------- lightning-persister/src/util.rs | 188 ------------ lightning/src/util/persist.rs | 51 ++-- 6 files changed, 94 insertions(+), 589 deletions(-) delete mode 100644 lightning-persister/src/util.rs diff --git a/bench/benches/bench.rs b/bench/benches/bench.rs index 54799f44c95..3fc3abe687b 100644 --- a/bench/benches/bench.rs +++ b/bench/benches/bench.rs @@ -15,7 +15,6 @@ criterion_group!(benches, lightning::routing::router::benches::generate_large_mpp_routes_with_probabilistic_scorer, lightning::sign::benches::bench_get_secure_random_bytes, lightning::ln::channelmanager::bench::bench_sends, - lightning_persister::bench::bench_sends, lightning_rapid_gossip_sync::bench::bench_reading_full_graph_from_file, lightning::routing::gossip::benches::read_network_graph, lightning::routing::gossip::benches::write_network_graph); diff --git a/lightning-background-processor/src/lib.rs b/lightning-background-processor/src/lib.rs index 8648920ec2c..353ed6738d6 100644 --- a/lightning-background-processor/src/lib.rs +++ b/lightning-background-processor/src/lib.rs @@ -500,9 +500,16 @@ use core::task; /// For example, in order to process background events in a [Tokio](https://tokio.rs/) task, you /// could setup `process_events_async` like this: /// ``` -/// # struct MyPersister {} -/// # impl lightning::util::persist::KVStorePersister for MyPersister { -/// # fn persist(&self, key: &str, object: &W) -> lightning::io::Result<()> { Ok(()) } +/// # use lightning::io; +/// # use std::sync::{Arc, Mutex}; +/// # use std::sync::atomic::{AtomicBool, Ordering}; +/// # use lightning_background_processor::{process_events_async, GossipSync}; +/// # struct MyStore {} +/// # impl lightning::util::persist::KVStore for MyStore { +/// # fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result> { Ok(Vec::new()) } +/// # fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) } +/// # fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) } +/// # fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result> { Ok(Vec::new()) } /// # } /// # struct MyEventHandler {} /// # impl MyEventHandler { @@ -514,23 +521,20 @@ use core::task; /// # fn send_data(&mut self, _data: &[u8], _resume_read: bool) -> usize { 0 } /// # fn disconnect_socket(&mut self) {} /// # } -/// # use std::sync::{Arc, Mutex}; -/// # use std::sync::atomic::{AtomicBool, Ordering}; -/// # use lightning_background_processor::{process_events_async, GossipSync}; /// # type MyBroadcaster = dyn lightning::chain::chaininterface::BroadcasterInterface + Send + Sync; /// # type MyFeeEstimator = dyn lightning::chain::chaininterface::FeeEstimator + Send + Sync; /// # type MyNodeSigner = dyn lightning::sign::NodeSigner + Send + Sync; /// # type MyUtxoLookup = dyn lightning::routing::utxo::UtxoLookup + Send + Sync; /// # type MyFilter = dyn lightning::chain::Filter + Send + Sync; /// # type MyLogger = dyn lightning::util::logger::Logger + Send + Sync; -/// # type MyChainMonitor = lightning::chain::chainmonitor::ChainMonitor, Arc, Arc, Arc, Arc>; +/// # type MyChainMonitor = lightning::chain::chainmonitor::ChainMonitor, Arc, Arc, Arc, Arc>; /// # type MyPeerManager = lightning::ln::peer_handler::SimpleArcPeerManager; /// # type MyNetworkGraph = lightning::routing::gossip::NetworkGraph>; /// # type MyGossipSync = lightning::routing::gossip::P2PGossipSync, Arc, Arc>; /// # type MyChannelManager = lightning::ln::channelmanager::SimpleArcChannelManager; /// # type MyScorer = Mutex, Arc>>; /// -/// # async fn setup_background_processing(my_persister: Arc, my_event_handler: Arc, my_chain_monitor: Arc, my_channel_manager: Arc, my_gossip_sync: Arc, my_logger: Arc, my_scorer: Arc, my_peer_manager: Arc) { +/// # async fn setup_background_processing(my_persister: Arc, my_event_handler: Arc, my_chain_monitor: Arc, my_channel_manager: Arc, my_gossip_sync: Arc, my_logger: Arc, my_scorer: Arc, my_peer_manager: Arc) { /// let background_persister = Arc::clone(&my_persister); /// let background_event_handler = Arc::clone(&my_event_handler); /// let background_chain_mon = Arc::clone(&my_chain_monitor); @@ -866,8 +870,8 @@ mod tests { use lightning::util::config::UserConfig; use lightning::util::ser::Writeable; use lightning::util::test_utils; - use lightning::util::persist::KVStorePersister; - use lightning_persister::FilesystemPersister; + use lightning::util::persist::{KVStore, CHANNEL_MANAGER_PERSISTENCE_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY, SCORER_PERSISTENCE_NAMESPACE, SCORER_PERSISTENCE_SUB_NAMESPACE, SCORER_PERSISTENCE_KEY}; + use lightning_persister::fs_store::FilesystemStore; use std::collections::VecDeque; use std::{fs, env}; use std::path::PathBuf; @@ -906,7 +910,7 @@ mod tests { >, Arc>; - type ChainMonitor = chainmonitor::ChainMonitor, Arc, Arc, Arc, Arc>; + type ChainMonitor = chainmonitor::ChainMonitor, Arc, Arc, Arc, Arc>; type PGS = Arc>>, Arc, Arc>>; type RGS = Arc>>, Arc>>; @@ -917,7 +921,7 @@ mod tests { rapid_gossip_sync: RGS, peer_manager: Arc, Arc, IgnoringMessageHandler, Arc, IgnoringMessageHandler, Arc>>, chain_monitor: Arc, - persister: Arc, + kv_store: Arc, tx_broadcaster: Arc, network_graph: Arc>>, logger: Arc, @@ -941,9 +945,9 @@ mod tests { impl Drop for Node { fn drop(&mut self) { - let data_dir = self.persister.get_data_dir(); + let data_dir = self.kv_store.get_data_dir(); match fs::remove_dir_all(data_dir.clone()) { - Err(e) => println!("Failed to remove test persister directory {}: {}", data_dir, e), + Err(e) => println!("Failed to remove test store directory {}: {}", data_dir.display(), e), _ => {} } } @@ -954,13 +958,13 @@ mod tests { graph_persistence_notifier: Option>, manager_error: Option<(std::io::ErrorKind, &'static str)>, scorer_error: Option<(std::io::ErrorKind, &'static str)>, - filesystem_persister: FilesystemPersister, + kv_store: FilesystemStore, } impl Persister { - fn new(data_dir: String) -> Self { - let filesystem_persister = FilesystemPersister::new(data_dir); - Self { graph_error: None, graph_persistence_notifier: None, manager_error: None, scorer_error: None, filesystem_persister } + fn new(data_dir: PathBuf) -> Self { + let kv_store = FilesystemStore::new(data_dir); + Self { graph_error: None, graph_persistence_notifier: None, manager_error: None, scorer_error: None, kv_store } } fn with_graph_error(self, error: std::io::ErrorKind, message: &'static str) -> Self { @@ -980,15 +984,25 @@ mod tests { } } - impl KVStorePersister for Persister { - fn persist(&self, key: &str, object: &W) -> std::io::Result<()> { - if key == "manager" { + impl KVStore for Persister { + fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> lightning::io::Result> { + self.kv_store.read(namespace, sub_namespace, key) + } + + fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> { + if namespace == CHANNEL_MANAGER_PERSISTENCE_NAMESPACE && + sub_namespace == CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE && + key == CHANNEL_MANAGER_PERSISTENCE_KEY + { if let Some((error, message)) = self.manager_error { return Err(std::io::Error::new(error, message)) } } - if key == "network_graph" { + if namespace == NETWORK_GRAPH_PERSISTENCE_NAMESPACE && + sub_namespace == NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE && + key == NETWORK_GRAPH_PERSISTENCE_KEY + { if let Some(sender) = &self.graph_persistence_notifier { match sender.send(()) { Ok(()) => {}, @@ -1001,13 +1015,24 @@ mod tests { } } - if key == "scorer" { + if namespace == SCORER_PERSISTENCE_NAMESPACE && + sub_namespace == SCORER_PERSISTENCE_SUB_NAMESPACE && + key == SCORER_PERSISTENCE_KEY + { if let Some((error, message)) = self.scorer_error { return Err(std::io::Error::new(error, message)) } } - self.filesystem_persister.persist(key, object) + self.kv_store.write(namespace, sub_namespace, key, buf) + } + + fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> { + self.kv_store.remove(namespace, sub_namespace, key, lazy) + } + + fn list(&self, namespace: &str, sub_namespace: &str) -> lightning::io::Result> { + self.kv_store.list(namespace, sub_namespace) } } @@ -1157,10 +1182,10 @@ mod tests { let seed = [i as u8; 32]; let router = Arc::new(DefaultRouter::new(network_graph.clone(), logger.clone(), seed, scorer.clone(), ())); let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Bitcoin)); - let persister = Arc::new(FilesystemPersister::new(format!("{}_persister_{}", &persist_dir, i))); + let kv_store = Arc::new(FilesystemStore::new(format!("{}_persister_{}", &persist_dir, i).into())); let now = Duration::from_secs(genesis_block.header.time as u64); let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_nanos())); - let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new(Some(chain_source.clone()), tx_broadcaster.clone(), logger.clone(), fee_estimator.clone(), persister.clone())); + let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new(Some(chain_source.clone()), tx_broadcaster.clone(), logger.clone(), fee_estimator.clone(), kv_store.clone())); let best_block = BestBlock::from_network(network); let params = ChainParameters { network, best_block }; let manager = Arc::new(ChannelManager::new(fee_estimator.clone(), chain_monitor.clone(), tx_broadcaster.clone(), router.clone(), logger.clone(), keys_manager.clone(), keys_manager.clone(), keys_manager.clone(), UserConfig::default(), params, genesis_block.header.time)); @@ -1172,7 +1197,7 @@ mod tests { onion_message_handler: IgnoringMessageHandler{}, custom_message_handler: IgnoringMessageHandler{} }; let peer_manager = Arc::new(PeerManager::new(msg_handler, 0, &seed, logger.clone(), keys_manager.clone())); - let node = Node { node: manager, p2p_gossip_sync, rapid_gossip_sync, peer_manager, chain_monitor, persister, tx_broadcaster, network_graph, logger, best_block, scorer }; + let node = Node { node: manager, p2p_gossip_sync, rapid_gossip_sync, peer_manager, chain_monitor, kv_store, tx_broadcaster, network_graph, logger, best_block, scorer }; nodes.push(node); } @@ -1267,7 +1292,7 @@ mod tests { let tx = open_channel!(nodes[0], nodes[1], 100000); // Initiate the background processors to watch each node. - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir)); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].p2p_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1332,7 +1357,7 @@ mod tests { // `ChainMonitor::rebroadcast_pending_claims` is called every `REBROADCAST_TIMER`, and // `PeerManager::timer_tick_occurred` every `PING_TIMER`. let (_, nodes) = create_nodes(1, "test_timer_tick_called"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir)); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1359,7 +1384,7 @@ mod tests { let (_, nodes) = create_nodes(2, "test_persist_error"); open_channel!(nodes[0], nodes[1], 100000); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_manager_error(std::io::ErrorKind::Other, "test")); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1379,7 +1404,7 @@ mod tests { let (_, nodes) = create_nodes(2, "test_persist_error_sync"); open_channel!(nodes[0], nodes[1], 100000); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_manager_error(std::io::ErrorKind::Other, "test")); let bp_future = super::process_events_async( @@ -1405,7 +1430,7 @@ mod tests { fn test_network_graph_persist_error() { // Test that if we encounter an error during network graph persistence, an error gets returned. let (_, nodes) = create_nodes(2, "test_persist_network_graph_error"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_graph_error(std::io::ErrorKind::Other, "test")); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].p2p_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1423,7 +1448,7 @@ mod tests { fn test_scorer_persist_error() { // Test that if we encounter an error during scorer persistence, an error gets returned. let (_, nodes) = create_nodes(2, "test_persist_scorer_error"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_scorer_error(std::io::ErrorKind::Other, "test")); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1441,7 +1466,7 @@ mod tests { fn test_background_event_handling() { let (_, mut nodes) = create_nodes(2, "test_background_event_handling"); let channel_value = 100000; - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir.clone())); // Set up a background event handler for FundingGenerationReady events. @@ -1514,7 +1539,7 @@ mod tests { #[test] fn test_scorer_persistence() { let (_, nodes) = create_nodes(2, "test_scorer_persistence"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir)); let event_handler = |_: _| {}; let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1586,7 +1611,7 @@ mod tests { let (sender, receiver) = std::sync::mpsc::sync_channel(1); let (_, nodes) = create_nodes(2, "test_not_pruning_network_graph_until_graph_sync_completion"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_graph_persistence_notifier(sender)); let event_handler = |_: _| {}; @@ -1605,7 +1630,7 @@ mod tests { let (sender, receiver) = std::sync::mpsc::sync_channel(1); let (_, nodes) = create_nodes(2, "test_not_pruning_network_graph_until_graph_sync_completion_async"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir).with_graph_persistence_notifier(sender)); let (exit_sender, exit_receiver) = tokio::sync::watch::channel(()); @@ -1745,7 +1770,7 @@ mod tests { }; let (_, nodes) = create_nodes(1, "test_payment_path_scoring"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir)); let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone())); @@ -1778,7 +1803,7 @@ mod tests { }; let (_, nodes) = create_nodes(1, "test_payment_path_scoring_async"); - let data_dir = nodes[0].persister.get_data_dir(); + let data_dir = nodes[0].kv_store.get_data_dir(); let persister = Arc::new(Persister::new(data_dir)); let (exit_sender, exit_receiver) = tokio::sync::watch::channel(()); diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index 271f3b882b3..7ba4223227d 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -16,10 +16,8 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] bitcoin = "0.29.0" lightning = { version = "0.0.116", path = "../lightning" } -libc = "0.2" [target.'cfg(windows)'.dependencies] -winapi = { version = "0.3", features = ["winbase"] } windows-sys = { version = "0.48.0", default-features = false, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] } [target.'cfg(ldk_bench)'.dependencies] diff --git a/lightning-persister/src/lib.rs b/lightning-persister/src/lib.rs index 932e4f41ad3..ae258e137d7 100644 --- a/lightning-persister/src/lib.rs +++ b/lightning-persister/src/lib.rs @@ -16,341 +16,3 @@ mod utils; #[cfg(test)] mod test_utils; - -mod util; - -extern crate lightning; -extern crate bitcoin; -extern crate libc; - -use bitcoin::hash_types::{BlockHash, Txid}; -use bitcoin::hashes::hex::FromHex; -use lightning::chain::channelmonitor::ChannelMonitor; -use lightning::sign::{EntropySource, SignerProvider}; -use lightning::util::ser::{ReadableArgs, Writeable}; -use lightning::util::persist::KVStorePersister; -use std::fs; -use std::io::Cursor; -use std::ops::Deref; -use std::path::{Path, PathBuf}; - -/// FilesystemPersister persists channel data on disk, where each channel's -/// data is stored in a file named after its funding outpoint. -/// -/// Warning: this module does the best it can with calls to persist data, but it -/// can only guarantee that the data is passed to the drive. It is up to the -/// drive manufacturers to do the actual persistence properly, which they often -/// don't (especially on consumer-grade hardware). Therefore, it is up to the -/// user to validate their entire storage stack, to ensure the writes are -/// persistent. -/// Corollary: especially when dealing with larger amounts of money, it is best -/// practice to have multiple channel data backups and not rely only on one -/// FilesystemPersister. -pub struct FilesystemPersister { - path_to_channel_data: String, -} - -impl FilesystemPersister { - /// Initialize a new FilesystemPersister and set the path to the individual channels' - /// files. - pub fn new(path_to_channel_data: String) -> Self { - Self { - path_to_channel_data, - } - } - - /// Get the directory which was provided when this persister was initialized. - pub fn get_data_dir(&self) -> String { - self.path_to_channel_data.clone() - } - - /// Read `ChannelMonitor`s from disk. - pub fn read_channelmonitors ( - &self, entropy_source: ES, signer_provider: SP - ) -> std::io::Result::Signer>)>> - where - ES::Target: EntropySource + Sized, - SP::Target: SignerProvider + Sized - { - let mut path = PathBuf::from(&self.path_to_channel_data); - path.push("monitors"); - if !Path::new(&path).exists() { - return Ok(Vec::new()); - } - let mut res = Vec::new(); - for file_option in fs::read_dir(path)? { - let file = file_option.unwrap(); - let owned_file_name = file.file_name(); - let filename = owned_file_name.to_str() - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, - "File name is not a valid utf8 string"))?; - if !filename.is_ascii() || filename.len() < 65 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid ChannelMonitor file name", - )); - } - if filename.ends_with(".tmp") { - // If we were in the middle of committing an new update and crashed, it should be - // safe to ignore the update - we should never have returned to the caller and - // irrevocably committed to the new state in any way. - continue; - } - - let txid: Txid = Txid::from_hex(filename.split_at(64).0) - .map_err(|_| std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx ID in filename", - ))?; - - let index: u16 = filename.split_at(65).1.parse() - .map_err(|_| std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Invalid tx index in filename", - ))?; - - let contents = fs::read(&file.path())?; - let mut buffer = Cursor::new(&contents); - match <(BlockHash, ChannelMonitor<::Signer>)>::read(&mut buffer, (&*entropy_source, &*signer_provider)) { - Ok((blockhash, channel_monitor)) => { - if channel_monitor.get_funding_txo().0.txid != txid || channel_monitor.get_funding_txo().0.index != index { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, - "ChannelMonitor was stored in the wrong file")); - } - res.push((blockhash, channel_monitor)); - } - Err(e) => return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Failed to deserialize ChannelMonitor: {}", e), - )) - } - } - Ok(res) - } -} - -impl KVStorePersister for FilesystemPersister { - fn persist(&self, key: &str, object: &W) -> std::io::Result<()> { - let mut dest_file = PathBuf::from(self.path_to_channel_data.clone()); - dest_file.push(key); - util::write_to_file(dest_file, object) - } -} - -#[cfg(test)] -mod tests { - extern crate lightning; - extern crate bitcoin; - use crate::FilesystemPersister; - use bitcoin::hashes::hex::FromHex; - use bitcoin::Txid; - use lightning::chain::ChannelMonitorUpdateStatus; - use lightning::chain::chainmonitor::Persist; - use lightning::chain::channelmonitor::CLOSED_CHANNEL_UPDATE_ID; - use lightning::chain::transaction::OutPoint; - use lightning::{check_closed_broadcast, check_closed_event, check_added_monitors}; - use lightning::events::{ClosureReason, MessageSendEventsProvider}; - use lightning::ln::functional_test_utils::*; - use lightning::util::test_utils; - use std::fs; - #[cfg(target_os = "windows")] - use { - lightning::get_event_msg, - lightning::ln::msgs::ChannelMessageHandler, - }; - - impl Drop for FilesystemPersister { - fn drop(&mut self) { - // We test for invalid directory names, so it's OK if directory removal - // fails. - match fs::remove_dir_all(&self.path_to_channel_data) { - Err(e) => println!("Failed to remove test persister directory: {}", e), - _ => {} - } - } - } - - #[test] - fn test_if_monitors_is_not_dir() { - let persister = FilesystemPersister::new("test_monitors_is_not_dir".to_string()); - - fs::create_dir_all(&persister.path_to_channel_data).unwrap(); - let mut path = std::path::PathBuf::from(&persister.path_to_channel_data); - path.push("monitors"); - fs::File::create(path).unwrap(); - - let chanmon_cfgs = create_chanmon_cfgs(1); - let mut node_cfgs = create_node_cfgs(1, &chanmon_cfgs); - let chain_mon_0 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[0].chain_source), &chanmon_cfgs[0].tx_broadcaster, &chanmon_cfgs[0].logger, &chanmon_cfgs[0].fee_estimator, &persister, node_cfgs[0].keys_manager); - node_cfgs[0].chain_monitor = chain_mon_0; - let node_chanmgrs = create_node_chanmgrs(1, &node_cfgs, &[None]); - let nodes = create_network(1, &node_cfgs, &node_chanmgrs); - - // Check that read_channelmonitors() returns error if monitors/ is not a - // directory. - assert!(persister.read_channelmonitors(nodes[0].keys_manager, nodes[0].keys_manager).is_err()); - } - - // Integration-test the FilesystemPersister. Test relaying a few payments - // and check that the persisted data is updated the appropriate number of - // times. - #[test] - fn test_filesystem_persister() { - // Create the nodes, giving them FilesystemPersisters for data persisters. - let persister_0 = FilesystemPersister::new("test_filesystem_persister_0".to_string()); - let persister_1 = FilesystemPersister::new("test_filesystem_persister_1".to_string()); - let chanmon_cfgs = create_chanmon_cfgs(2); - let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let chain_mon_0 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[0].chain_source), &chanmon_cfgs[0].tx_broadcaster, &chanmon_cfgs[0].logger, &chanmon_cfgs[0].fee_estimator, &persister_0, node_cfgs[0].keys_manager); - let chain_mon_1 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[1].chain_source), &chanmon_cfgs[1].tx_broadcaster, &chanmon_cfgs[1].logger, &chanmon_cfgs[1].fee_estimator, &persister_1, node_cfgs[1].keys_manager); - node_cfgs[0].chain_monitor = chain_mon_0; - node_cfgs[1].chain_monitor = chain_mon_1; - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - - // Check that the persisted channel data is empty before any channels are - // open. - let mut persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager, nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 0); - let mut persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager, nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 0); - - // Helper to make sure the channel is on the expected update ID. - macro_rules! check_persisted_data { - ($expected_update_id: expr) => { - persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager, nodes[0].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_0.len(), 1); - for (_, mon) in persisted_chan_data_0.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager, nodes[1].keys_manager).unwrap(); - assert_eq!(persisted_chan_data_1.len(), 1); - for (_, mon) in persisted_chan_data_1.iter() { - assert_eq!(mon.get_latest_update_id(), $expected_update_id); - } - } - } - - // Create some initial channel and check that a channel was persisted. - let _ = create_announced_chan_between_nodes(&nodes, 0, 1); - check_persisted_data!(0); - - // Send a few payments and make sure the monitors are updated to the latest. - send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000); - check_persisted_data!(5); - send_payment(&nodes[1], &vec!(&nodes[0])[..], 4000000); - check_persisted_data!(10); - - // Force close because cooperative close doesn't result in any persisted - // updates. - nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap(); - check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000); - check_closed_broadcast!(nodes[0], true); - check_added_monitors!(nodes[0], 1); - - let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); - assert_eq!(node_txn.len(), 1); - - connect_block(&nodes[1], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[0].clone(), node_txn[0].clone()])); - check_closed_broadcast!(nodes[1], true); - check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000); - check_added_monitors!(nodes[1], 1); - - // Make sure everything is persisted as expected after close. - check_persisted_data!(CLOSED_CHANNEL_UPDATE_ID); - } - - // Test that if the persister's path to channel data is read-only, writing a - // monitor to it results in the persister returning a PermanentFailure. - // Windows ignores the read-only flag for folders, so this test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir_perm_failure() { - let persister = FilesystemPersister::new("test_readonly_dir_perm_failure".to_string()); - fs::create_dir_all(&persister.path_to_channel_data).unwrap(); - - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1); - nodes[1].node.force_close_broadcasting_latest_txn(&chan.2, &nodes[0].node.get_our_node_id()).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed, [nodes[0].node.get_our_node_id()], 100000); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Set the persister's directory to read-only, which should result in - // returning a permanent failure when we then attempt to persist a - // channel update. - let path = &persister.path_to_channel_data; - let mut perms = fs::metadata(path).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path, perms).unwrap(); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0 - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - ChannelMonitorUpdateStatus::PermanentFailure => {}, - _ => panic!("unexpected result from persisting new channel") - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } - - // Test that if a persister's directory name is invalid, monitor persistence - // will fail. - #[cfg(target_os = "windows")] - #[test] - fn test_fail_on_open() { - // Set up a dummy channel and force close. This will produce a monitor - // that we can then use to test persistence. - let chanmon_cfgs = create_chanmon_cfgs(2); - let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); - let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); - let nodes = create_network(2, &node_cfgs, &node_chanmgrs); - let chan = create_announced_chan_between_nodes(&nodes, 0, 1); - nodes[1].node.force_close_broadcasting_latest_txn(&chan.2, &nodes[0].node.get_our_node_id()).unwrap(); - check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed, [nodes[0].node.get_our_node_id()], 100000); - let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); - let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); - let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); - - // Create the persister with an invalid directory name and test that the - // channel fails to open because the directories fail to be created. There - // don't seem to be invalid filename characters on Unix that Rust doesn't - // handle, hence why the test is Windows-only. - let persister = FilesystemPersister::new(":<>/".to_string()); - - let test_txo = OutPoint { - txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), - index: 0 - }; - match persister.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { - ChannelMonitorUpdateStatus::PermanentFailure => {}, - _ => panic!("unexpected result from persisting new channel") - } - - nodes[1].node.get_and_clear_pending_msg_events(); - added_monitors.clear(); - } -} - -#[cfg(ldk_bench)] -/// Benches -pub mod bench { - use criterion::Criterion; - - /// Bench! - pub fn bench_sends(bench: &mut Criterion) { - let persister_a = super::FilesystemPersister::new("bench_filesystem_persister_a".to_string()); - let persister_b = super::FilesystemPersister::new("bench_filesystem_persister_b".to_string()); - lightning::ln::channelmanager::bench::bench_two_sends( - bench, "bench_filesystem_persisted_sends", persister_a, persister_b); - } -} diff --git a/lightning-persister/src/util.rs b/lightning-persister/src/util.rs deleted file mode 100644 index 20c4a815185..00000000000 --- a/lightning-persister/src/util.rs +++ /dev/null @@ -1,188 +0,0 @@ -#[cfg(target_os = "windows")] -extern crate winapi; - -use std::fs; -use std::path::PathBuf; -use std::io::BufWriter; - -#[cfg(not(target_os = "windows"))] -use std::os::unix::io::AsRawFd; - -use lightning::util::ser::Writeable; - -#[cfg(target_os = "windows")] -use { - std::ffi::OsStr, - std::os::windows::ffi::OsStrExt -}; - -#[cfg(target_os = "windows")] -macro_rules! call { - ($e: expr) => ( - if $e != 0 { - return Ok(()) - } else { - return Err(std::io::Error::last_os_error()) - } - ) -} - -#[cfg(target_os = "windows")] -fn path_to_windows_str>(path: T) -> Vec { - path.as_ref().encode_wide().chain(Some(0)).collect() -} - -#[allow(bare_trait_objects)] -pub(crate) fn write_to_file(dest_file: PathBuf, data: &W) -> std::io::Result<()> { - let mut tmp_file = dest_file.clone(); - tmp_file.set_extension("tmp"); - - let parent_directory = dest_file.parent().unwrap(); - fs::create_dir_all(parent_directory)?; - // Do a crazy dance with lots of fsync()s to be overly cautious here... - // We never want to end up in a state where we've lost the old data, or end up using the - // old data on power loss after we've returned. - // The way to atomically write a file on Unix platforms is: - // open(tmpname), write(tmpfile), fsync(tmpfile), close(tmpfile), rename(), fsync(dir) - { - // Note that going by rust-lang/rust@d602a6b, on MacOS it is only safe to use - // rust stdlib 1.36 or higher. - let mut buf = BufWriter::new(fs::File::create(&tmp_file)?); - data.write(&mut buf)?; - buf.into_inner()?.sync_all()?; - } - // Fsync the parent directory on Unix. - #[cfg(not(target_os = "windows"))] - { - fs::rename(&tmp_file, &dest_file)?; - let dir_file = fs::OpenOptions::new().read(true).open(parent_directory)?; - unsafe { libc::fsync(dir_file.as_raw_fd()); } - } - #[cfg(target_os = "windows")] - { - if dest_file.exists() { - unsafe {winapi::um::winbase::ReplaceFileW( - path_to_windows_str(dest_file).as_ptr(), path_to_windows_str(tmp_file).as_ptr(), std::ptr::null(), - winapi::um::winbase::REPLACEFILE_IGNORE_MERGE_ERRORS, - std::ptr::null_mut() as *mut winapi::ctypes::c_void, - std::ptr::null_mut() as *mut winapi::ctypes::c_void - )}; - } else { - call!(unsafe {winapi::um::winbase::MoveFileExW( - path_to_windows_str(tmp_file).as_ptr(), path_to_windows_str(dest_file).as_ptr(), - winapi::um::winbase::MOVEFILE_WRITE_THROUGH | winapi::um::winbase::MOVEFILE_REPLACE_EXISTING - )}); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use lightning::util::ser::{Writer, Writeable}; - - use super::{write_to_file}; - use std::fs; - use std::io; - use std::path::PathBuf; - - struct TestWriteable{} - impl Writeable for TestWriteable { - fn write(&self, writer: &mut W) -> Result<(), std::io::Error> { - writer.write_all(&[42; 1]) - } - } - - // Test that if the persister's path to channel data is read-only, writing - // data to it fails. Windows ignores the read-only flag for folders, so this - // test is Unix-only. - #[cfg(not(target_os = "windows"))] - #[test] - fn test_readonly_dir() { - let test_writeable = TestWriteable{}; - let filename = "test_readonly_dir_persister_filename".to_string(); - let path = "test_readonly_dir_persister_dir"; - fs::create_dir_all(path).unwrap(); - let mut perms = fs::metadata(path).unwrap().permissions(); - perms.set_readonly(true); - fs::set_permissions(path, perms).unwrap(); - let mut dest_file = PathBuf::from(path); - dest_file.push(filename); - match write_to_file(dest_file, &test_writeable) { - Err(e) => assert_eq!(e.kind(), io::ErrorKind::PermissionDenied), - _ => panic!("Unexpected error message") - } - } - - // Test failure to rename in the process of atomically creating a channel - // monitor's file. We induce this failure by making the `tmp` file a - // directory. - // Explanation: given "from" = the file being renamed, "to" = the destination - // file that already exists: Unix should fail because if "from" is a file, - // then "to" is also required to be a file. - // TODO: ideally try to make this work on Windows again - #[cfg(not(target_os = "windows"))] - #[test] - fn test_rename_failure() { - let test_writeable = TestWriteable{}; - let filename = "test_rename_failure_filename"; - let path = "test_rename_failure_dir"; - let mut dest_file = PathBuf::from(path); - dest_file.push(filename); - // Create the channel data file and make it a directory. - fs::create_dir_all(dest_file.clone()).unwrap(); - match write_to_file(dest_file, &test_writeable) { - Err(e) => assert_eq!(e.raw_os_error(), Some(libc::EISDIR)), - _ => panic!("Unexpected Ok(())") - } - fs::remove_dir_all(path).unwrap(); - } - - #[test] - fn test_diskwriteable_failure() { - struct FailingWriteable {} - impl Writeable for FailingWriteable { - fn write(&self, _writer: &mut W) -> Result<(), std::io::Error> { - Err(std::io::Error::new(std::io::ErrorKind::Other, "expected failure")) - } - } - - let filename = "test_diskwriteable_failure"; - let path = "test_diskwriteable_failure_dir"; - let test_writeable = FailingWriteable{}; - let mut dest_file = PathBuf::from(path); - dest_file.push(filename); - match write_to_file(dest_file, &test_writeable) { - Err(e) => { - assert_eq!(e.kind(), std::io::ErrorKind::Other); - assert_eq!(e.get_ref().unwrap().to_string(), "expected failure"); - }, - _ => panic!("unexpected result") - } - fs::remove_dir_all(path).unwrap(); - } - - // Test failure to create the temporary file in the persistence process. - // We induce this failure by having the temp file already exist and be a - // directory. - #[test] - fn test_tmp_file_creation_failure() { - let test_writeable = TestWriteable{}; - let filename = "test_tmp_file_creation_failure_filename".to_string(); - let path = "test_tmp_file_creation_failure_dir"; - let mut dest_file = PathBuf::from(path); - dest_file.push(filename); - let mut tmp_file = dest_file.clone(); - tmp_file.set_extension("tmp"); - fs::create_dir_all(tmp_file).unwrap(); - match write_to_file(dest_file, &test_writeable) { - Err(e) => { - #[cfg(not(target_os = "windows"))] - assert_eq!(e.raw_os_error(), Some(libc::EISDIR)); - #[cfg(target_os = "windows")] - assert_eq!(e.kind(), io::ErrorKind::PermissionDenied); - } - _ => panic!("Unexpected error message") - } - } -} diff --git a/lightning/src/util/persist.rs b/lightning/src/util/persist.rs index 5b122b7a570..ca0605c9598 100644 --- a/lightning/src/util/persist.rs +++ b/lightning/src/util/persist.rs @@ -114,15 +114,6 @@ pub trait KVStore { fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result>; } -/// Trait for a key-value store for persisting some writeable object at some key -/// Implementing `KVStorePersister` provides auto-implementations for [`Persister`] -/// and [`Persist`] traits. It uses "manager", "network_graph", -/// and "monitors/{funding_txo_id}_{funding_txo_index}" for keys. -pub trait KVStorePersister { - /// Persist the given writeable using the provided key - fn persist(&self, key: &str, object: &W) -> io::Result<()>; -} - /// Trait that handles persisting a [`ChannelManager`], [`NetworkGraph`], and [`WriteableScore`] to disk. pub trait Persister<'a, M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: Deref, R: Deref, L: Deref, S: WriteableScore<'a>> where M::Target: 'static + chain::Watch<::Signer>, @@ -144,7 +135,8 @@ pub trait Persister<'a, M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: fn persist_scorer(&self, scorer: &S) -> Result<(), io::Error>; } -impl<'a, A: KVStorePersister, M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: Deref, R: Deref, L: Deref, S: WriteableScore<'a>> Persister<'a, M, T, ES, NS, SP, F, R, L, S> for A + +impl<'a, A: KVStore, M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: Deref, R: Deref, L: Deref, S: WriteableScore<'a>> Persister<'a, M, T, ES, NS, SP, F, R, L, S> for A where M::Target: 'static + chain::Watch<::Signer>, T::Target: 'static + BroadcasterInterface, ES::Target: 'static + EntropySource, @@ -154,39 +146,56 @@ impl<'a, A: KVStorePersister, M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Dere R::Target: 'static + Router, L::Target: 'static + Logger, { - /// Persist the given ['ChannelManager'] to disk with the name "manager", returning an error if persistence failed. + /// Persist the given [`ChannelManager`] to disk, returning an error if persistence failed. fn persist_manager(&self, channel_manager: &ChannelManager) -> Result<(), io::Error> { - self.persist("manager", channel_manager) + self.write(CHANNEL_MANAGER_PERSISTENCE_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + &channel_manager.encode()) } - /// Persist the given [`NetworkGraph`] to disk with the name "network_graph", returning an error if persistence failed. + /// Persist the given [`NetworkGraph`] to disk, returning an error if persistence failed. fn persist_graph(&self, network_graph: &NetworkGraph) -> Result<(), io::Error> { - self.persist("network_graph", network_graph) + self.write(NETWORK_GRAPH_PERSISTENCE_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + &network_graph.encode()) } - /// Persist the given [`WriteableScore`] to disk with name "scorer", returning an error if persistence failed. + /// Persist the given [`WriteableScore`] to disk, returning an error if persistence failed. fn persist_scorer(&self, scorer: &S) -> Result<(), io::Error> { - self.persist("scorer", &scorer) + self.write(SCORER_PERSISTENCE_NAMESPACE, + SCORER_PERSISTENCE_SUB_NAMESPACE, + SCORER_PERSISTENCE_KEY, + &scorer.encode()) } } -impl Persist for K { +impl Persist for K { // TODO: We really need a way for the persister to inform the user that its time to crash/shut // down once these start returning failure. // A PermanentFailure implies we should probably just shut down the node since we're // force-closing channels without even broadcasting! fn persist_new_channel(&self, funding_txo: OutPoint, monitor: &ChannelMonitor, _update_id: MonitorUpdateId) -> chain::ChannelMonitorUpdateStatus { - let key = format!("monitors/{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - match self.persist(&key, monitor) { + let key = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + match self.write( + CHANNEL_MONITOR_PERSISTENCE_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SUB_NAMESPACE, + &key, &monitor.encode()) + { Ok(()) => chain::ChannelMonitorUpdateStatus::Completed, Err(_) => chain::ChannelMonitorUpdateStatus::PermanentFailure, } } fn update_persisted_channel(&self, funding_txo: OutPoint, _update: Option<&ChannelMonitorUpdate>, monitor: &ChannelMonitor, _update_id: MonitorUpdateId) -> chain::ChannelMonitorUpdateStatus { - let key = format!("monitors/{}_{}", funding_txo.txid.to_hex(), funding_txo.index); - match self.persist(&key, monitor) { + let key = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index); + match self.write( + CHANNEL_MONITOR_PERSISTENCE_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SUB_NAMESPACE, + &key, &monitor.encode()) + { Ok(()) => chain::ChannelMonitorUpdateStatus::Completed, Err(_) => chain::ChannelMonitorUpdateStatus::PermanentFailure, } From 20433deceabc18b7948fb35ad7ca4ca749ea51b3 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 4 Aug 2023 16:27:39 +0200 Subject: [PATCH 28/38] Migrate `FilesystemPersister` tests to `FilesystemStore` --- lightning-persister/Cargo.toml | 1 + lightning-persister/src/fs_store.rs | 140 +++++++++++++++++++++++++- lightning-persister/src/test_utils.rs | 73 +++++++++++++- 3 files changed, 212 insertions(+), 2 deletions(-) diff --git a/lightning-persister/Cargo.toml b/lightning-persister/Cargo.toml index 7ba4223227d..361ab0c57a1 100644 --- a/lightning-persister/Cargo.toml +++ b/lightning-persister/Cargo.toml @@ -25,3 +25,4 @@ criterion = { version = "0.4", optional = true, default-features = false } [dev-dependencies] lightning = { version = "0.0.116", path = "../lightning", features = ["_test_utils"] } +bitcoin = { version = "0.29.0", default-features = false } diff --git a/lightning-persister/src/fs_store.rs b/lightning-persister/src/fs_store.rs index f74806d7d8e..638e74e6506 100644 --- a/lightning-persister/src/fs_store.rs +++ b/lightning-persister/src/fs_store.rs @@ -367,7 +367,36 @@ impl KVStore for FilesystemStore { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::do_read_write_remove_list_persist; + use crate::test_utils::{do_read_write_remove_list_persist, do_test_store}; + + use bitcoin::hashes::hex::FromHex; + use bitcoin::Txid; + + use lightning::chain::ChannelMonitorUpdateStatus; + use lightning::chain::chainmonitor::Persist; + use lightning::chain::transaction::OutPoint; + use lightning::check_closed_event; + use lightning::events::{ClosureReason, MessageSendEventsProvider}; + use lightning::ln::functional_test_utils::*; + use lightning::util::test_utils; + use lightning::util::persist::read_channel_monitors; + use std::fs; + #[cfg(target_os = "windows")] + use { + lightning::get_event_msg, + lightning::ln::msgs::ChannelMessageHandler, + }; + + impl Drop for FilesystemStore { + fn drop(&mut self) { + // We test for invalid directory names, so it's OK if directory removal + // fails. + match fs::remove_dir_all(&self.data_dir) { + Err(e) => println!("Failed to remove test persister directory: {}", e), + _ => {} + } + } + } #[test] fn read_write_remove_list_persist() { @@ -376,4 +405,113 @@ mod tests { let fs_store = FilesystemStore::new(temp_path); do_read_write_remove_list_persist(&fs_store); } + + #[test] + fn test_if_monitors_is_not_dir() { + let store = FilesystemStore::new("test_monitors_is_not_dir".into()); + + fs::create_dir_all(&store.get_data_dir()).unwrap(); + let mut path = std::path::PathBuf::from(&store.get_data_dir()); + path.push("monitors"); + fs::File::create(path).unwrap(); + + let chanmon_cfgs = create_chanmon_cfgs(1); + let mut node_cfgs = create_node_cfgs(1, &chanmon_cfgs); + let chain_mon_0 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[0].chain_source), &chanmon_cfgs[0].tx_broadcaster, &chanmon_cfgs[0].logger, &chanmon_cfgs[0].fee_estimator, &store, node_cfgs[0].keys_manager); + node_cfgs[0].chain_monitor = chain_mon_0; + let node_chanmgrs = create_node_chanmgrs(1, &node_cfgs, &[None]); + let nodes = create_network(1, &node_cfgs, &node_chanmgrs); + + // Check that read_channel_monitors() returns error if monitors/ is not a + // directory. + assert!(read_channel_monitors(&store, nodes[0].keys_manager, nodes[0].keys_manager).is_err()); + } + + #[test] + fn test_filesystem_store() { + // Create the nodes, giving them FilesystemStores for data stores. + let store_0 = FilesystemStore::new("test_filesystem_store_0".into()); + let store_1 = FilesystemStore::new("test_filesystem_store_1".into()); + do_test_store(&store_0, &store_1) + } + + // Test that if the store's path to channel data is read-only, writing a + // monitor to it results in the store returning a PermanentFailure. + // Windows ignores the read-only flag for folders, so this test is Unix-only. + #[cfg(not(target_os = "windows"))] + #[test] + fn test_readonly_dir_perm_failure() { + let store = FilesystemStore::new("test_readonly_dir_perm_failure".into()); + fs::create_dir_all(&store.get_data_dir()).unwrap(); + + // Set up a dummy channel and force close. This will produce a monitor + // that we can then use to test persistence. + let chanmon_cfgs = create_chanmon_cfgs(2); + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + let chan = create_announced_chan_between_nodes(&nodes, 0, 1); + nodes[1].node.force_close_broadcasting_latest_txn(&chan.2, &nodes[0].node.get_our_node_id()).unwrap(); + check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed, [nodes[0].node.get_our_node_id()], 100000); + let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); + let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); + let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); + + // Set the store's directory to read-only, which should result in + // returning a permanent failure when we then attempt to persist a + // channel update. + let path = &store.get_data_dir(); + let mut perms = fs::metadata(path).unwrap().permissions(); + perms.set_readonly(true); + fs::set_permissions(path, perms).unwrap(); + + let test_txo = OutPoint { + txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), + index: 0 + }; + match store.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { + ChannelMonitorUpdateStatus::PermanentFailure => {}, + _ => panic!("unexpected result from persisting new channel") + } + + nodes[1].node.get_and_clear_pending_msg_events(); + added_monitors.clear(); + } + + // Test that if a store's directory name is invalid, monitor persistence + // will fail. + #[cfg(target_os = "windows")] + #[test] + fn test_fail_on_open() { + // Set up a dummy channel and force close. This will produce a monitor + // that we can then use to test persistence. + let chanmon_cfgs = create_chanmon_cfgs(2); + let node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + let chan = create_announced_chan_between_nodes(&nodes, 0, 1); + nodes[1].node.force_close_broadcasting_latest_txn(&chan.2, &nodes[0].node.get_our_node_id()).unwrap(); + check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed, [nodes[0].node.get_our_node_id()], 100000); + let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap(); + let update_map = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap(); + let update_id = update_map.get(&added_monitors[0].0.to_channel_id()).unwrap(); + + // Create the store with an invalid directory name and test that the + // channel fails to open because the directories fail to be created. There + // don't seem to be invalid filename characters on Unix that Rust doesn't + // handle, hence why the test is Windows-only. + let store = FilesystemStore::new(":<>/".into()); + + let test_txo = OutPoint { + txid: Txid::from_hex("8984484a580b825b9972d7adb15050b3ab624ccd731946b3eeddb92f4e7ef6be").unwrap(), + index: 0 + }; + match store.persist_new_channel(test_txo, &added_monitors[0].1, update_id.2) { + ChannelMonitorUpdateStatus::PermanentFailure => {}, + _ => panic!("unexpected result from persisting new channel") + } + + nodes[1].node.get_and_clear_pending_msg_events(); + added_monitors.clear(); + } } diff --git a/lightning-persister/src/test_utils.rs b/lightning-persister/src/test_utils.rs index 357a8b2ee8c..91557500f3d 100644 --- a/lightning-persister/src/test_utils.rs +++ b/lightning-persister/src/test_utils.rs @@ -1,4 +1,11 @@ -use lightning::util::persist::{KVStore, KVSTORE_NAMESPACE_KEY_MAX_LEN}; +use lightning::util::persist::{KVStore, KVSTORE_NAMESPACE_KEY_MAX_LEN, read_channel_monitors}; +use lightning::ln::functional_test_utils::{connect_block, create_announced_chan_between_nodes, + create_chanmon_cfgs, create_dummy_block, create_network, create_node_cfgs, create_node_chanmgrs, + send_payment}; +use lightning::chain::channelmonitor::CLOSED_CHANNEL_UPDATE_ID; +use lightning::util::test_utils; +use lightning::{check_closed_broadcast, check_closed_event, check_added_monitors}; +use lightning::events::ClosureReason; use std::panic::RefUnwindSafe; @@ -48,3 +55,67 @@ pub(crate) fn do_read_write_remove_list_persist(kv_s let listed_keys = kv_store.list(&max_chars, &max_chars).unwrap(); assert_eq!(listed_keys.len(), 0); } + +// Integration-test the given KVStore implementation. Test relaying a few payments and check that +// the persisted data is updated the appropriate number of times. +pub(crate) fn do_test_store(store_0: &K, store_1: &K) { + let chanmon_cfgs = create_chanmon_cfgs(2); + let mut node_cfgs = create_node_cfgs(2, &chanmon_cfgs); + let chain_mon_0 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[0].chain_source), &chanmon_cfgs[0].tx_broadcaster, &chanmon_cfgs[0].logger, &chanmon_cfgs[0].fee_estimator, store_0, node_cfgs[0].keys_manager); + let chain_mon_1 = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[1].chain_source), &chanmon_cfgs[1].tx_broadcaster, &chanmon_cfgs[1].logger, &chanmon_cfgs[1].fee_estimator, store_1, node_cfgs[1].keys_manager); + node_cfgs[0].chain_monitor = chain_mon_0; + node_cfgs[1].chain_monitor = chain_mon_1; + let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]); + let nodes = create_network(2, &node_cfgs, &node_chanmgrs); + + // Check that the persisted channel data is empty before any channels are + // open. + let mut persisted_chan_data_0 = read_channel_monitors(store_0, nodes[0].keys_manager, nodes[0].keys_manager).unwrap(); + assert_eq!(persisted_chan_data_0.len(), 0); + let mut persisted_chan_data_1 = read_channel_monitors(store_1, nodes[1].keys_manager, nodes[1].keys_manager).unwrap(); + assert_eq!(persisted_chan_data_1.len(), 0); + + // Helper to make sure the channel is on the expected update ID. + macro_rules! check_persisted_data { + ($expected_update_id: expr) => { + persisted_chan_data_0 = read_channel_monitors(store_0, nodes[0].keys_manager, nodes[0].keys_manager).unwrap(); + assert_eq!(persisted_chan_data_0.len(), 1); + for (_, mon) in persisted_chan_data_0.iter() { + assert_eq!(mon.get_latest_update_id(), $expected_update_id); + } + persisted_chan_data_1 = read_channel_monitors(store_1, nodes[1].keys_manager, nodes[1].keys_manager).unwrap(); + assert_eq!(persisted_chan_data_1.len(), 1); + for (_, mon) in persisted_chan_data_1.iter() { + assert_eq!(mon.get_latest_update_id(), $expected_update_id); + } + } + } + + // Create some initial channel and check that a channel was persisted. + let _ = create_announced_chan_between_nodes(&nodes, 0, 1); + check_persisted_data!(0); + + // Send a few payments and make sure the monitors are updated to the latest. + send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000); + check_persisted_data!(5); + send_payment(&nodes[1], &vec!(&nodes[0])[..], 4000000); + check_persisted_data!(10); + + // Force close because cooperative close doesn't result in any persisted + // updates. + nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap(); + check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000); + check_closed_broadcast!(nodes[0], true); + check_added_monitors!(nodes[0], 1); + + let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap(); + assert_eq!(node_txn.len(), 1); + + connect_block(&nodes[1], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[0].clone(), node_txn[0].clone()])); + check_closed_broadcast!(nodes[1], true); + check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000); + check_added_monitors!(nodes[1], 1); + + // Make sure everything is persisted as expected after close. + check_persisted_data!(CLOSED_CHANNEL_UPDATE_ID); +} From bc277713a926072bc647ac17fc926d35079f71a4 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Fri, 4 Aug 2023 11:34:45 +0200 Subject: [PATCH 29/38] Add benchmarking for `FilesystemStore` We re-add benchmarking for `FilesystemStore` now that we switched over to it. --- bench/benches/bench.rs | 1 + lightning-persister/src/fs_store.rs | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/bench/benches/bench.rs b/bench/benches/bench.rs index 3fc3abe687b..bc4bd010822 100644 --- a/bench/benches/bench.rs +++ b/bench/benches/bench.rs @@ -15,6 +15,7 @@ criterion_group!(benches, lightning::routing::router::benches::generate_large_mpp_routes_with_probabilistic_scorer, lightning::sign::benches::bench_get_secure_random_bytes, lightning::ln::channelmanager::bench::bench_sends, + lightning_persister::fs_store::bench::bench_sends, lightning_rapid_gossip_sync::bench::bench_reading_full_graph_from_file, lightning::routing::gossip::benches::read_network_graph, lightning::routing::gossip::benches::write_network_graph); diff --git a/lightning-persister/src/fs_store.rs b/lightning-persister/src/fs_store.rs index 638e74e6506..56d071da9f0 100644 --- a/lightning-persister/src/fs_store.rs +++ b/lightning-persister/src/fs_store.rs @@ -515,3 +515,17 @@ mod tests { added_monitors.clear(); } } + +#[cfg(ldk_bench)] +/// Benches +pub mod bench { + use criterion::Criterion; + + /// Bench! + pub fn bench_sends(bench: &mut Criterion) { + let store_a = super::FilesystemStore::new("bench_filesystem_store_a".into()); + let store_b = super::FilesystemStore::new("bench_filesystem_store_b".into()); + lightning::ln::channelmanager::bench::bench_two_sends( + bench, "bench_filesystem_persisted_sends", store_a, store_b); + } +} From d731a3542eb837890d2cd1850da573c73a2f8189 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Mon, 21 Aug 2023 16:17:35 +0200 Subject: [PATCH 30/38] Add `TestStore` implementation of `KVStore` --- lightning/src/util/test_utils.rs | 92 ++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index 7a9ce06910b..8e2be87d8be 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -38,6 +38,7 @@ use crate::util::config::UserConfig; use crate::util::test_channel_signer::{TestChannelSigner, EnforcementState}; use crate::util::logger::{Logger, Level, Record}; use crate::util::ser::{Readable, ReadableArgs, Writer, Writeable}; +use crate::util::persist::KVStore; use bitcoin::EcdsaSighashType; use bitcoin::blockdata::constants::ChainHash; @@ -425,6 +426,97 @@ impl chainmonitor::Persist fo } } +pub(crate) struct TestStore { + persisted_bytes: Mutex>>>, + read_only: bool, +} + +impl TestStore { + pub fn new(read_only: bool) -> Self { + let persisted_bytes = Mutex::new(HashMap::new()); + Self { persisted_bytes, read_only } + } +} + +impl KVStore for TestStore { + fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result> { + let persisted_lock = self.persisted_bytes.lock().unwrap(); + let prefixed = if sub_namespace.is_empty() { + namespace.to_string() + } else { + format!("{}/{}", namespace, sub_namespace) + }; + + if let Some(outer_ref) = persisted_lock.get(&prefixed) { + if let Some(inner_ref) = outer_ref.get(key) { + let bytes = inner_ref.clone(); + Ok(bytes) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Key not found")) + } + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "Namespace not found")) + } + } + + fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { + if self.read_only { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "Cannot modify read-only store", + )); + } + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = if sub_namespace.is_empty() { + namespace.to_string() + } else { + format!("{}/{}", namespace, sub_namespace) + }; + let outer_e = persisted_lock.entry(prefixed).or_insert(HashMap::new()); + let mut bytes = Vec::new(); + bytes.write_all(buf)?; + outer_e.insert(key.to_string(), bytes); + Ok(()) + } + + fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, _lazy: bool) -> io::Result<()> { + if self.read_only { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "Cannot modify read-only store", + )); + } + + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = if sub_namespace.is_empty() { + namespace.to_string() + } else { + format!("{}/{}", namespace, sub_namespace) + }; + if let Some(outer_ref) = persisted_lock.get_mut(&prefixed) { + outer_ref.remove(&key.to_string()); + } + + Ok(()) + } + + fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result> { + let mut persisted_lock = self.persisted_bytes.lock().unwrap(); + + let prefixed = if sub_namespace.is_empty() { + namespace.to_string() + } else { + format!("{}/{}", namespace, sub_namespace) + }; + match persisted_lock.entry(prefixed) { + hash_map::Entry::Occupied(e) => Ok(e.get().keys().cloned().collect()), + hash_map::Entry::Vacant(_) => Ok(Vec::new()), + } + } +} + pub struct TestBroadcaster { pub txn_broadcasted: Mutex>, pub blocks: Arc>>, From 446e0781966de7b64c7179b16a0a8d6d458aec79 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Tue, 18 Jul 2023 14:41:50 -0500 Subject: [PATCH 31/38] Add an InvoiceRequestFailed event When an invoice is requested but either receives an error or never receives a response, surface an event to indicate to the user that the corresponding future payment has failed. --- lightning/src/events/mod.rs | 25 +++++++++++++++++++ lightning/src/ln/channelmanager.rs | 20 +++++++++++---- .../invoice_request_failed_downgrade.txt | 3 +++ 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 pending_changelog/invoice_request_failed_downgrade.txt diff --git a/lightning/src/events/mod.rs b/lightning/src/events/mod.rs index f4f7a7cca9a..bb98e271597 100644 --- a/lightning/src/events/mod.rs +++ b/lightning/src/events/mod.rs @@ -508,6 +508,14 @@ pub enum Event { /// serialized prior to LDK version 0.0.117. sender_intended_total_msat: Option, }, + /// Indicates a request for an invoice failed to yield a response in a reasonable amount of time + /// or was explicitly abandoned by [`ChannelManager::abandon_payment`]. + /// + /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment + InvoiceRequestFailed { + /// The `payment_id` to have been associated with payment for the requested invoice. + payment_id: PaymentId, + }, /// Indicates an outbound payment we made succeeded (i.e. it made it all the way to its target /// and we got back the payment preimage for it). /// @@ -1148,6 +1156,12 @@ impl Writeable for Event { (8, funding_txo, required), }); }, + &Event::InvoiceRequestFailed { ref payment_id } => { + 33u8.write(writer)?; + write_tlv_fields!(writer, { + (0, payment_id, required), + }) + }, // Note that, going forward, all new events must only write data inside of // `write_tlv_fields`. Versions 0.0.101+ will ignore odd-numbered events that write // data via `write_tlv_fields`. @@ -1535,6 +1549,17 @@ impl MaybeReadable for Event { }; f() }, + 33u8 => { + let f = || { + _init_and_read_len_prefixed_tlv_fields!(reader, { + (0, payment_id, required), + }); + Ok(Some(Event::InvoiceRequestFailed { + payment_id: payment_id.0.unwrap(), + })) + }; + f() + }, // Versions prior to 0.0.100 did not ignore odd types, instead returning InvalidValue. // Version 0.0.100 failed to properly ignore odd types, possibly resulting in corrupt // reads. diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index d620d03e426..743b435842f 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -3381,10 +3381,12 @@ where } - /// Signals that no further retries for the given payment should occur. Useful if you have a + /// Signals that no further attempts for the given payment should occur. Useful if you have a /// pending outbound payment with retries remaining, but wish to stop retrying the payment before /// retries are exhausted. /// + /// # Event Generation + /// /// If no [`Event::PaymentFailed`] event had been generated before, one will be generated as soon /// as there are no remaining pending HTLCs for this payment. /// @@ -3392,11 +3394,19 @@ where /// wait until you receive either a [`Event::PaymentFailed`] or [`Event::PaymentSent`] event to /// determine the ultimate status of a payment. /// - /// If an [`Event::PaymentFailed`] event is generated and we restart without this - /// [`ChannelManager`] having been persisted, another [`Event::PaymentFailed`] may be generated. + /// # Requested Invoices /// - /// [`Event::PaymentFailed`]: events::Event::PaymentFailed - /// [`Event::PaymentSent`]: events::Event::PaymentSent + /// In the case of paying a [`Bolt12Invoice`], abandoning the payment prior to receiving the + /// invoice will result in an [`Event::InvoiceRequestFailed`] and prevent any attempts at paying + /// it once received. The other events may only be generated once the invoice has been received. + /// + /// # Restart Behavior + /// + /// If an [`Event::PaymentFailed`] is generated and we restart without first persisting the + /// [`ChannelManager`], another [`Event::PaymentFailed`] may be generated; likewise for + /// [`Event::InvoiceRequestFailed`]. + /// + /// [`Bolt12Invoice`]: crate::offers::invoice::Bolt12Invoice pub fn abandon_payment(&self, payment_id: PaymentId) { let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(self); self.pending_outbound_payments.abandon_payment(payment_id, PaymentFailureReason::UserAbandoned, &self.pending_events); diff --git a/pending_changelog/invoice_request_failed_downgrade.txt b/pending_changelog/invoice_request_failed_downgrade.txt new file mode 100644 index 00000000000..d701cef04b5 --- /dev/null +++ b/pending_changelog/invoice_request_failed_downgrade.txt @@ -0,0 +1,3 @@ +## Backwards Compatibility + +* If an `Event::InvoiceRequestFailed` was generated for a BOLT 12 payment (#2371), downgrading will result in the payment silently failing if the event had not been processed yet. From 5c4b7ad8cf010961d31f3a62f640ce18b88e61ea Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Mon, 17 Jul 2023 16:55:22 -0500 Subject: [PATCH 32/38] Add PendingOutboundPayment::AwaitingInvoice When a BOLT 12 invoice has been requested, in order to guarantee invoice payment idempotency the future payment needs to be tracked. Add an AwaitingInvoice variant to PendingOutboundPayment such that only requested invoices are paid only once. Timeout after a few timer ticks if a request has not been received. --- lightning/src/ln/channelmanager.rs | 15 ++++- lightning/src/ln/outbound_payment.rs | 98 +++++++++++++++++++++++----- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 743b435842f..21fdad65540 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -1343,7 +1343,7 @@ pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3; /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the /// idempotency of payments by [`PaymentId`]. See -/// [`OutboundPayments::remove_stale_resolved_payments`]. +/// [`OutboundPayments::remove_stale_payments`]. pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7; /// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is disconnected @@ -1688,6 +1688,11 @@ pub enum ChannelShutdownState { /// These include payments that have yet to find a successful path, or have unresolved HTLCs. #[derive(Debug, PartialEq)] pub enum RecentPaymentDetails { + /// When an invoice was requested but not yet received, and thus a payment has not been sent. + AwaitingInvoice { + /// Identifier for the payment to ensure idempotency. + payment_id: PaymentId, + }, /// When a payment is still being sent and awaiting successful delivery. Pending { /// Hash of the payment that is currently being sent but has yet to be fulfilled or @@ -2419,7 +2424,10 @@ where /// [`Event::PaymentSent`]: events::Event::PaymentSent pub fn list_recent_payments(&self) -> Vec { self.pending_outbound_payments.pending_outbound_payments.lock().unwrap().iter() - .filter_map(|(_, pending_outbound_payment)| match pending_outbound_payment { + .filter_map(|(payment_id, pending_outbound_payment)| match pending_outbound_payment { + PendingOutboundPayment::AwaitingInvoice { .. } => { + Some(RecentPaymentDetails::AwaitingInvoice { payment_id: *payment_id }) + }, PendingOutboundPayment::Retryable { payment_hash, total_msat, .. } => { Some(RecentPaymentDetails::Pending { payment_hash: *payment_hash, @@ -4665,7 +4673,7 @@ where let _ = handle_error!(self, err, counterparty_node_id); } - self.pending_outbound_payments.remove_stale_resolved_payments(&self.pending_events); + self.pending_outbound_payments.remove_stale_payments(&self.pending_events); // Technically we don't need to do this here, but if we have holding cell entries in a // channel that need freeing, it's better to do that here and block a background task @@ -8363,6 +8371,7 @@ where session_priv.write(writer)?; } } + PendingOutboundPayment::AwaitingInvoice { .. } => {}, PendingOutboundPayment::Fulfilled { .. } => {}, PendingOutboundPayment::Abandoned { .. } => {}, } diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 67d90d2dbf8..f5341b88405 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -32,12 +32,21 @@ use core::ops::Deref; use crate::prelude::*; use crate::sync::Mutex; +/// The number of ticks of [`ChannelManager::timer_tick_occurred`] until an invoice request without +/// a response is timed out. +/// +/// [`ChannelManager::timer_tick_occurred`]: crate::ln::channelmanager::ChannelManager::timer_tick_occurred +const INVOICE_REQUEST_TIMEOUT_TICKS: u8 = 3; + /// Stores the session_priv for each part of a payment that is still pending. For versions 0.0.102 /// and later, also stores information for retrying the payment. pub(crate) enum PendingOutboundPayment { Legacy { session_privs: HashSet<[u8; 32]>, }, + AwaitingInvoice { + timer_ticks_without_response: u8, + }, Retryable { retry_strategy: Option, attempts: PaymentAttempts, @@ -108,6 +117,12 @@ impl PendingOutboundPayment { params.previously_failed_channels.push(scid); } } + fn is_awaiting_invoice(&self) -> bool { + match self { + PendingOutboundPayment::AwaitingInvoice { .. } => true, + _ => false, + } + } pub(super) fn is_fulfilled(&self) -> bool { match self { PendingOutboundPayment::Fulfilled { .. } => true, @@ -130,6 +145,7 @@ impl PendingOutboundPayment { fn payment_hash(&self) -> Option { match self { PendingOutboundPayment::Legacy { .. } => None, + PendingOutboundPayment::AwaitingInvoice { .. } => None, PendingOutboundPayment::Retryable { payment_hash, .. } => Some(*payment_hash), PendingOutboundPayment::Fulfilled { payment_hash, .. } => *payment_hash, PendingOutboundPayment::Abandoned { payment_hash, .. } => Some(*payment_hash), @@ -142,8 +158,11 @@ impl PendingOutboundPayment { PendingOutboundPayment::Legacy { session_privs } | PendingOutboundPayment::Retryable { session_privs, .. } | PendingOutboundPayment::Fulfilled { session_privs, .. } | - PendingOutboundPayment::Abandoned { session_privs, .. } - => session_privs, + PendingOutboundPayment::Abandoned { session_privs, .. } => session_privs, + PendingOutboundPayment::AwaitingInvoice { .. } => { + debug_assert!(false); + return; + }, }); let payment_hash = self.payment_hash(); *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash, timer_ticks_without_htlcs: 0 }; @@ -169,7 +188,11 @@ impl PendingOutboundPayment { PendingOutboundPayment::Fulfilled { session_privs, .. } | PendingOutboundPayment::Abandoned { session_privs, .. } => { session_privs.remove(session_priv) - } + }, + PendingOutboundPayment::AwaitingInvoice { .. } => { + debug_assert!(false); + false + }, }; if remove_res { if let PendingOutboundPayment::Retryable { ref mut pending_amt_msat, ref mut pending_fee_msat, .. } = self { @@ -189,6 +212,10 @@ impl PendingOutboundPayment { PendingOutboundPayment::Retryable { session_privs, .. } => { session_privs.insert(session_priv) } + PendingOutboundPayment::AwaitingInvoice { .. } => { + debug_assert!(false); + false + }, PendingOutboundPayment::Fulfilled { .. } => false, PendingOutboundPayment::Abandoned { .. } => false, }; @@ -210,7 +237,8 @@ impl PendingOutboundPayment { PendingOutboundPayment::Fulfilled { session_privs, .. } | PendingOutboundPayment::Abandoned { session_privs, .. } => { session_privs.len() - } + }, + PendingOutboundPayment::AwaitingInvoice { .. } => 0, } } } @@ -679,7 +707,7 @@ impl OutboundPayments { let mut outbounds = self.pending_outbound_payments.lock().unwrap(); outbounds.retain(|pmt_id, pmt| { let mut retain = true; - if !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 { + if !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 && !pmt.is_awaiting_invoice() { pmt.mark_abandoned(PaymentFailureReason::RetriesExhausted); if let PendingOutboundPayment::Abandoned { payment_hash, reason, .. } = pmt { pending_events.lock().unwrap().push_back((events::Event::PaymentFailed { @@ -697,7 +725,8 @@ impl OutboundPayments { pub(super) fn needs_abandon(&self) -> bool { let outbounds = self.pending_outbound_payments.lock().unwrap(); outbounds.iter().any(|(_, pmt)| - !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 && !pmt.is_fulfilled()) + !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 && !pmt.is_fulfilled() && + !pmt.is_awaiting_invoice()) } /// Errors immediately on [`RetryableSendFailure`] error conditions. Otherwise, further errors may @@ -845,6 +874,10 @@ impl OutboundPayments { log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); return }, + PendingOutboundPayment::AwaitingInvoice { .. } => { + log_error!(logger, "Payment not yet sent"); + return + }, PendingOutboundPayment::Fulfilled { .. } => { log_error!(logger, "Payment already completed"); return @@ -1051,6 +1084,21 @@ impl OutboundPayments { } } + #[allow(unused)] + pub(super) fn add_new_awaiting_invoice(&self, payment_id: PaymentId) -> Result<(), ()> { + let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap(); + match pending_outbounds.entry(payment_id) { + hash_map::Entry::Occupied(_) => Err(()), + hash_map::Entry::Vacant(entry) => { + entry.insert(PendingOutboundPayment::AwaitingInvoice { + timer_ticks_without_response: 0, + }); + + Ok(()) + }, + } + } + fn pay_route_internal( &self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, keysend_preimage: Option, payment_id: PaymentId, recv_value_msat: Option, @@ -1256,19 +1304,19 @@ impl OutboundPayments { } } - pub(super) fn remove_stale_resolved_payments(&self, - pending_events: &Mutex)>>) + pub(super) fn remove_stale_payments( + &self, pending_events: &Mutex)>>) { - // If an outbound payment was completed, and no pending HTLCs remain, we should remove it - // from the map. However, if we did that immediately when the last payment HTLC is claimed, - // this could race the user making a duplicate send_payment call and our idempotency - // guarantees would be violated. Instead, we wait a few timer ticks to do the actual - // removal. This should be more than sufficient to ensure the idempotency of any - // `send_payment` calls that were made at the same time the `PaymentSent` event was being - // processed. let mut pending_outbound_payments = self.pending_outbound_payments.lock().unwrap(); - let pending_events = pending_events.lock().unwrap(); + let mut pending_events = pending_events.lock().unwrap(); pending_outbound_payments.retain(|payment_id, payment| { + // If an outbound payment was completed, and no pending HTLCs remain, we should remove it + // from the map. However, if we did that immediately when the last payment HTLC is claimed, + // this could race the user making a duplicate send_payment call and our idempotency + // guarantees would be violated. Instead, we wait a few timer ticks to do the actual + // removal. This should be more than sufficient to ensure the idempotency of any + // `send_payment` calls that were made at the same time the `PaymentSent` event was being + // processed. if let PendingOutboundPayment::Fulfilled { session_privs, timer_ticks_without_htlcs, .. } = payment { let mut no_remaining_entries = session_privs.is_empty(); if no_remaining_entries { @@ -1293,6 +1341,16 @@ impl OutboundPayments { *timer_ticks_without_htlcs = 0; true } + } else if let PendingOutboundPayment::AwaitingInvoice { timer_ticks_without_response, .. } = payment { + *timer_ticks_without_response += 1; + if *timer_ticks_without_response <= INVOICE_REQUEST_TIMEOUT_TICKS { + true + } else { + pending_events.push_back( + (events::Event::InvoiceRequestFailed { payment_id: *payment_id }, None) + ); + false + } } else { true } }); } @@ -1438,6 +1496,11 @@ impl OutboundPayments { }, None)); payment.remove(); } + } else if let PendingOutboundPayment::AwaitingInvoice { .. } = payment.get() { + pending_events.lock().unwrap().push_back((events::Event::InvoiceRequestFailed { + payment_id, + }, None)); + payment.remove(); } } } @@ -1501,6 +1564,9 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment, (1, reason, option), (2, payment_hash, required), }, + (5, AwaitingInvoice) => { + (0, timer_ticks_without_response, required), + }, ); #[cfg(test)] From 1f2d0e7649a1fcb0e20a9b530f805b2be1e880b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 6 Sep 2023 13:56:46 -0500 Subject: [PATCH 33/38] Move IDEMPOTENCY_TIMEOUT_TICKS to where it is used --- lightning/src/ln/channelmanager.rs | 5 ----- lightning/src/ln/outbound_payment.rs | 8 +++++++- lightning/src/ln/payment_tests.rs | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 21fdad65540..d83d654d9c4 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -1341,11 +1341,6 @@ const CHECK_CLTV_EXPIRY_SANITY_2: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_G /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until expiry of incomplete MPPs pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3; -/// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the -/// idempotency of payments by [`PaymentId`]. See -/// [`OutboundPayments::remove_stale_payments`]. -pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7; - /// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is disconnected /// until we mark the channel disabled and gossip the update. pub(crate) const DISABLE_GOSSIP_TICKS: u8 = 10; diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index f5341b88405..aa92043cf91 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -16,7 +16,7 @@ use bitcoin::secp256k1::{self, Secp256k1, SecretKey}; use crate::sign::{EntropySource, NodeSigner, Recipient}; use crate::events::{self, PaymentFailureReason}; use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use crate::ln::channelmanager::{ChannelDetails, EventCompletionAction, HTLCSource, IDEMPOTENCY_TIMEOUT_TICKS, PaymentId}; +use crate::ln::channelmanager::{ChannelDetails, EventCompletionAction, HTLCSource, PaymentId}; use crate::ln::onion_utils::{DecodedOnionFailure, HTLCFailReason}; use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteParameters, Router}; use crate::util::errors::APIError; @@ -32,6 +32,12 @@ use core::ops::Deref; use crate::prelude::*; use crate::sync::Mutex; +/// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the idempotency +/// of payments by [`PaymentId`]. See [`OutboundPayments::remove_stale_payments`]. +/// +/// [`ChannelManager::timer_tick_occurred`]: crate::ln::channelmanager::ChannelManager::timer_tick_occurred +pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7; + /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until an invoice request without /// a response is timed out. /// diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index 0c94adb3560..506209df2b5 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -17,11 +17,11 @@ use crate::sign::EntropySource; use crate::chain::transaction::OutPoint; use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason, PaymentPurpose}; use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS; -use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo}; +use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo}; use crate::ln::features::Bolt11InvoiceFeatures; use crate::ln::{msgs, ChannelId, PaymentSecret, PaymentPreimage}; use crate::ln::msgs::ChannelMessageHandler; -use crate::ln::outbound_payment::Retry; +use crate::ln::outbound_payment::{IDEMPOTENCY_TIMEOUT_TICKS, Retry}; use crate::routing::gossip::{EffectiveCapacity, RoutingFees}; use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route}; use crate::routing::scoring::ChannelUsage; From 9c4739af60ee3782df70d8c8eafc7d162344810c Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 30 Aug 2023 20:22:18 -0500 Subject: [PATCH 34/38] Test for removing stale AwaitingInvoice payments --- lightning/src/ln/outbound_payment.rs | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index aa92043cf91..f8989366509 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -1585,7 +1585,7 @@ mod tests { use crate::ln::channelmanager::{PaymentId, RecipientOnionFields}; use crate::ln::features::{ChannelFeatures, NodeFeatures}; use crate::ln::msgs::{ErrorAction, LightningError}; - use crate::ln::outbound_payment::{OutboundPayments, Retry, RetryableSendFailure}; + use crate::ln::outbound_payment::{INVOICE_REQUEST_TIMEOUT_TICKS, OutboundPayments, Retry, RetryableSendFailure}; use crate::routing::gossip::NetworkGraph; use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteHop, RouteParameters}; use crate::sync::{Arc, Mutex, RwLock}; @@ -1782,4 +1782,35 @@ mod tests { } else { panic!("Unexpected event"); } if let Event::PaymentFailed { .. } = events[1].0 { } else { panic!("Unexpected event"); } } + + #[test] + fn removes_stale_awaiting_invoice() { + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + assert!(!outbound_payments.has_pending_payments()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + for _ in 0..INVOICE_REQUEST_TIMEOUT_TICKS { + outbound_payments.remove_stale_payments(&pending_events); + assert!(outbound_payments.has_pending_payments()); + assert!(pending_events.lock().unwrap().is_empty()); + } + + outbound_payments.remove_stale_payments(&pending_events); + assert!(!outbound_payments.has_pending_payments()); + assert!(!pending_events.lock().unwrap().is_empty()); + assert_eq!( + pending_events.lock().unwrap().pop_front(), + Some((Event::InvoiceRequestFailed { payment_id }, None)), + ); + assert!(pending_events.lock().unwrap().is_empty()); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_err()); + } } From 1ce341ba93ad21e0b73cee03a6a0dda73b7ed094 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 6 Sep 2023 15:17:01 -0500 Subject: [PATCH 35/38] Test removing abandoned AwaitingInvoice payments --- lightning/src/ln/outbound_payment.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index f8989366509..e154b3f0d73 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -1813,4 +1813,26 @@ mod tests { assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_err()); } + + #[test] + fn removes_abandoned_awaiting_invoice() { + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + assert!(!outbound_payments.has_pending_payments()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + outbound_payments.abandon_payment( + payment_id, PaymentFailureReason::UserAbandoned, &pending_events + ); + assert!(!outbound_payments.has_pending_payments()); + assert!(!pending_events.lock().unwrap().is_empty()); + assert_eq!( + pending_events.lock().unwrap().pop_front(), + Some((Event::InvoiceRequestFailed { payment_id }, None)), + ); + assert!(pending_events.lock().unwrap().is_empty()); + } } From 2e6a64072ba36cd5726312a9469dc0da9237bc40 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 30 Aug 2023 12:01:15 -0500 Subject: [PATCH 36/38] Add PendingOutboundPayment::InvoiceReceived When a BOLT 12 invoice has been received, a payment attempt is made and any errors result in abandoning the PendingOutboundPayment. This results in generating at PaymentFailed event, which has a PaymentHash. Thus, when receiving an invoice, transition from AwaitingInvoice to a new InvoiceReceived state, the latter of which contains a PaymentHash such the abandon_payment helper can still be used. --- lightning/src/ln/channelmanager.rs | 7 ++++- lightning/src/ln/outbound_payment.rs | 38 ++++++++++++++++++---------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index d83d654d9c4..f1f7a8a58e6 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -1683,7 +1683,7 @@ pub enum ChannelShutdownState { /// These include payments that have yet to find a successful path, or have unresolved HTLCs. #[derive(Debug, PartialEq)] pub enum RecentPaymentDetails { - /// When an invoice was requested but not yet received, and thus a payment has not been sent. + /// When an invoice was requested and thus a payment has not yet been sent. AwaitingInvoice { /// Identifier for the payment to ensure idempotency. payment_id: PaymentId, @@ -2423,6 +2423,10 @@ where PendingOutboundPayment::AwaitingInvoice { .. } => { Some(RecentPaymentDetails::AwaitingInvoice { payment_id: *payment_id }) }, + // InvoiceReceived is an intermediate state and doesn't need to be exposed + PendingOutboundPayment::InvoiceReceived { .. } => { + Some(RecentPaymentDetails::AwaitingInvoice { payment_id: *payment_id }) + }, PendingOutboundPayment::Retryable { payment_hash, total_msat, .. } => { Some(RecentPaymentDetails::Pending { payment_hash: *payment_hash, @@ -8367,6 +8371,7 @@ where } } PendingOutboundPayment::AwaitingInvoice { .. } => {}, + PendingOutboundPayment::InvoiceReceived { .. } => {}, PendingOutboundPayment::Fulfilled { .. } => {}, PendingOutboundPayment::Abandoned { .. } => {}, } diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index e154b3f0d73..cf60c2cf2fa 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -53,6 +53,9 @@ pub(crate) enum PendingOutboundPayment { AwaitingInvoice { timer_ticks_without_response: u8, }, + InvoiceReceived { + payment_hash: PaymentHash, + }, Retryable { retry_strategy: Option, attempts: PaymentAttempts, @@ -152,6 +155,7 @@ impl PendingOutboundPayment { match self { PendingOutboundPayment::Legacy { .. } => None, PendingOutboundPayment::AwaitingInvoice { .. } => None, + PendingOutboundPayment::InvoiceReceived { payment_hash } => Some(*payment_hash), PendingOutboundPayment::Retryable { payment_hash, .. } => Some(*payment_hash), PendingOutboundPayment::Fulfilled { payment_hash, .. } => *payment_hash, PendingOutboundPayment::Abandoned { payment_hash, .. } => Some(*payment_hash), @@ -165,10 +169,8 @@ impl PendingOutboundPayment { PendingOutboundPayment::Retryable { session_privs, .. } | PendingOutboundPayment::Fulfilled { session_privs, .. } | PendingOutboundPayment::Abandoned { session_privs, .. } => session_privs, - PendingOutboundPayment::AwaitingInvoice { .. } => { - debug_assert!(false); - return; - }, + PendingOutboundPayment::AwaitingInvoice { .. } | + PendingOutboundPayment::InvoiceReceived { .. } => { debug_assert!(false); return; }, }); let payment_hash = self.payment_hash(); *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash, timer_ticks_without_htlcs: 0 }; @@ -183,6 +185,12 @@ impl PendingOutboundPayment { payment_hash: *payment_hash, reason: Some(reason) }; + } else if let PendingOutboundPayment::InvoiceReceived { payment_hash } = self { + *self = PendingOutboundPayment::Abandoned { + session_privs: HashSet::new(), + payment_hash: *payment_hash, + reason: Some(reason) + }; } } @@ -195,10 +203,8 @@ impl PendingOutboundPayment { PendingOutboundPayment::Abandoned { session_privs, .. } => { session_privs.remove(session_priv) }, - PendingOutboundPayment::AwaitingInvoice { .. } => { - debug_assert!(false); - false - }, + PendingOutboundPayment::AwaitingInvoice { .. } | + PendingOutboundPayment::InvoiceReceived { .. } => { debug_assert!(false); false }, }; if remove_res { if let PendingOutboundPayment::Retryable { ref mut pending_amt_msat, ref mut pending_fee_msat, .. } = self { @@ -217,11 +223,9 @@ impl PendingOutboundPayment { PendingOutboundPayment::Legacy { session_privs } | PendingOutboundPayment::Retryable { session_privs, .. } => { session_privs.insert(session_priv) - } - PendingOutboundPayment::AwaitingInvoice { .. } => { - debug_assert!(false); - false - }, + }, + PendingOutboundPayment::AwaitingInvoice { .. } | + PendingOutboundPayment::InvoiceReceived { .. } => { debug_assert!(false); false }, PendingOutboundPayment::Fulfilled { .. } => false, PendingOutboundPayment::Abandoned { .. } => false, }; @@ -245,6 +249,7 @@ impl PendingOutboundPayment { session_privs.len() }, PendingOutboundPayment::AwaitingInvoice { .. } => 0, + PendingOutboundPayment::InvoiceReceived { .. } => 0, } } } @@ -880,7 +885,9 @@ impl OutboundPayments { log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); return }, - PendingOutboundPayment::AwaitingInvoice { .. } => { + PendingOutboundPayment::AwaitingInvoice { .. } | + PendingOutboundPayment::InvoiceReceived { .. } => + { log_error!(logger, "Payment not yet sent"); return }, @@ -1573,6 +1580,9 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment, (5, AwaitingInvoice) => { (0, timer_ticks_without_response, required), }, + (7, InvoiceReceived) => { + (0, payment_hash, required), + }, ); #[cfg(test)] From 1cfd3c50f8ddd64167064d9e2aa5fc8e4d11d438 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Fri, 1 Sep 2023 16:23:27 -0500 Subject: [PATCH 37/38] Refactor OutboundPayments::retry_payment_internal Consolidate the creation and insertion of onion_session_privs to the PendingOutboundPayment::Retryable arm. In an upcoming commit, this method will be reused for an initial BOLT 12 invoice payment. However, onion_session_privs are created using a different helper. --- lightning/src/ln/outbound_payment.rs | 53 ++++++++++++++++------------ 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index cf60c2cf2fa..ddf51cdd796 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -839,12 +839,6 @@ impl OutboundPayments { } } - const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; - let mut onion_session_privs = Vec::with_capacity(route.paths.len()); - for _ in 0..route.paths.len() { - onion_session_privs.push(entropy_source.get_secure_random_bytes()); - } - macro_rules! abandon_with_entry { ($payment: expr, $reason: expr) => { $payment.get_mut().mark_abandoned($reason); @@ -860,26 +854,49 @@ impl OutboundPayments { } } } - let (total_msat, recipient_onion, keysend_preimage) = { + let (total_msat, recipient_onion, keysend_preimage, onion_session_privs) = { let mut outbounds = self.pending_outbound_payments.lock().unwrap(); match outbounds.entry(payment_id) { hash_map::Entry::Occupied(mut payment) => { - let res = match payment.get() { + match payment.get() { PendingOutboundPayment::Retryable { total_msat, keysend_preimage, payment_secret, payment_metadata, custom_tlvs, pending_amt_msat, .. } => { + const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; let retry_amt_msat = route.get_total_amount(); if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 { log_error!(logger, "retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat); abandon_with_entry!(payment, PaymentFailureReason::UnexpectedError); return } - (*total_msat, RecipientOnionFields { - payment_secret: *payment_secret, - payment_metadata: payment_metadata.clone(), - custom_tlvs: custom_tlvs.clone(), - }, *keysend_preimage) + + if !payment.get().is_retryable_now() { + log_error!(logger, "Retries exhausted for payment id {}", &payment_id); + abandon_with_entry!(payment, PaymentFailureReason::RetriesExhausted); + return + } + + let total_msat = *total_msat; + let recipient_onion = RecipientOnionFields { + payment_secret: *payment_secret, + payment_metadata: payment_metadata.clone(), + custom_tlvs: custom_tlvs.clone(), + }; + let keysend_preimage = *keysend_preimage; + + let mut onion_session_privs = Vec::with_capacity(route.paths.len()); + for _ in 0..route.paths.len() { + onion_session_privs.push(entropy_source.get_secure_random_bytes()); + } + + for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { + assert!(payment.get_mut().insert(*session_priv_bytes, path)); + } + + payment.get_mut().increment_attempts(); + + (total_msat, recipient_onion, keysend_preimage, onion_session_privs) }, PendingOutboundPayment::Legacy { .. } => { log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); @@ -899,17 +916,7 @@ impl OutboundPayments { log_error!(logger, "Payment already abandoned (with some HTLCs still pending)"); return }, - }; - if !payment.get().is_retryable_now() { - log_error!(logger, "Retries exhausted for payment id {}", &payment_id); - abandon_with_entry!(payment, PaymentFailureReason::RetriesExhausted); - return - } - payment.get_mut().increment_attempts(); - for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { - assert!(payment.get_mut().insert(*session_priv_bytes, path)); } - res }, hash_map::Entry::Vacant(_) => { log_error!(logger, "Payment with ID {} not found", &payment_id); From eb8b14b77d77c93f08e0814d36d0e00292b2701d Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 6 Sep 2023 14:31:57 -0500 Subject: [PATCH 38/38] Rename OutboundPayments::retry_payment_internal It will be used for initial attempts at paying BOLT 12 invoices, so rename it something that covers both that and retries. Support paying BOLT 12 invoices Add a send_payment_for_bolt12_invoice method to OutboundPayments for initiating payment of a BOLT 12 invoice. This will be called from an OffersMessageHandler, after which any retries are handled using the Retryable logic. pub(crate) visibility for offers/test_utils.rs The test utilities for Offers are needed for testing message handling in ChannelManager and OutboundPayments. Add tests for send_payment_for_bolt12_invoice Use u32 instead of usize in Retry::Attempts An upcoming commit requires serializing Retry, so use a type with a fixed byte length. Otherwise, using eight bytes to serialize a usize would fail to read on 32-bit machines. Configure BOLT 12 invoice payment retry strategy Replace a constant three retry attempts for BOLT 12 invoice payments with a retry strategy specified when creating a pending outbound payment. This is configured by users in a later commit when constructing an InvoiceRequest or a Refund. Rename SocketAddress from NetAddress --- fuzz/src/fromstr_to_netaddress.rs | 4 +- lightning-net-tokio/src/lib.rs | 8 +- lightning/src/ln/msgs.rs | 180 ++++---- lightning/src/ln/outbound_payment.rs | 431 ++++++++++++++++-- lightning/src/ln/peer_handler.rs | 132 +++--- lightning/src/offers/invoice.rs | 2 +- lightning/src/offers/mod.rs | 2 +- lightning/src/offers/test_utils.rs | 20 +- lightning/src/routing/gossip.rs | 12 +- .../move_netaddress_to_socketaddress.txt | 1 + 10 files changed, 568 insertions(+), 224 deletions(-) create mode 100644 pending_changelog/move_netaddress_to_socketaddress.txt diff --git a/fuzz/src/fromstr_to_netaddress.rs b/fuzz/src/fromstr_to_netaddress.rs index 19984112369..dba2d44451e 100644 --- a/fuzz/src/fromstr_to_netaddress.rs +++ b/fuzz/src/fromstr_to_netaddress.rs @@ -7,7 +7,7 @@ // You may not use this file except in accordance with one or both of these // licenses. -use lightning::ln::msgs::NetAddress; +use lightning::ln::msgs::SocketAddress; use core::str::FromStr; use crate::utils::test_logger; @@ -15,7 +15,7 @@ use crate::utils::test_logger; #[inline] pub fn do_test(data: &[u8]) { if let Ok(s) = std::str::from_utf8(data) { - let _ = NetAddress::from_str(s); + let _ = SocketAddress::from_str(s); } } diff --git a/lightning-net-tokio/src/lib.rs b/lightning-net-tokio/src/lib.rs index 6e2ea3f14c1..06aed3194f9 100644 --- a/lightning-net-tokio/src/lib.rs +++ b/lightning-net-tokio/src/lib.rs @@ -39,7 +39,7 @@ use tokio::io::AsyncWrite; use lightning::ln::peer_handler; use lightning::ln::peer_handler::SocketDescriptor as LnSocketTrait; use lightning::ln::peer_handler::APeerManager; -use lightning::ln::msgs::NetAddress; +use lightning::ln::msgs::SocketAddress; use std::ops::Deref; use std::task::{self, Poll}; @@ -274,13 +274,13 @@ impl Connection { } } -fn get_addr_from_stream(stream: &StdTcpStream) -> Option { +fn get_addr_from_stream(stream: &StdTcpStream) -> Option { match stream.peer_addr() { - Ok(SocketAddr::V4(sockaddr)) => Some(NetAddress::IPv4 { + Ok(SocketAddr::V4(sockaddr)) => Some(SocketAddress::TcpIpV4 { addr: sockaddr.ip().octets(), port: sockaddr.port(), }), - Ok(SocketAddr::V6(sockaddr)) => Some(NetAddress::IPv6 { + Ok(SocketAddr::V6(sockaddr)) => Some(SocketAddress::TcpIpV6 { addr: sockaddr.ip().octets(), port: sockaddr.port(), }), diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index d12dafb65af..6bd5ec3f729 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -101,7 +101,7 @@ pub struct Init { /// message. A node can decide to use that information to discover a potential update to its /// public IPv4 address (NAT) and use that for a [`NodeAnnouncement`] update message containing /// the new address. - pub remote_network_address: Option, + pub remote_network_address: Option, } /// An [`error`] message to be sent to or received from a peer. @@ -749,16 +749,16 @@ pub struct AnnouncementSignatures { /// An address which can be used to connect to a remote peer. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum NetAddress { - /// An IPv4 address/port on which the peer is listening. - IPv4 { +pub enum SocketAddress { + /// An IPv4 address and port on which the peer is listening. + TcpIpV4 { /// The 4-byte IPv4 address addr: [u8; 4], /// The port on which the node is listening port: u16, }, - /// An IPv6 address/port on which the peer is listening. - IPv6 { + /// An IPv6 address and port on which the peer is listening. + TcpIpV6 { /// The 16-byte IPv6 address addr: [u8; 16], /// The port on which the node is listening @@ -791,28 +791,28 @@ pub enum NetAddress { port: u16, }, } -impl NetAddress { +impl SocketAddress { /// Gets the ID of this address type. Addresses in [`NodeAnnouncement`] messages should be sorted /// by this. pub(crate) fn get_id(&self) -> u8 { match self { - &NetAddress::IPv4 {..} => { 1 }, - &NetAddress::IPv6 {..} => { 2 }, - &NetAddress::OnionV2(_) => { 3 }, - &NetAddress::OnionV3 {..} => { 4 }, - &NetAddress::Hostname {..} => { 5 }, + &SocketAddress::TcpIpV4 {..} => { 1 }, + &SocketAddress::TcpIpV6 {..} => { 2 }, + &SocketAddress::OnionV2(_) => { 3 }, + &SocketAddress::OnionV3 {..} => { 4 }, + &SocketAddress::Hostname {..} => { 5 }, } } /// Strict byte-length of address descriptor, 1-byte type not recorded fn len(&self) -> u16 { match self { - &NetAddress::IPv4 { .. } => { 6 }, - &NetAddress::IPv6 { .. } => { 18 }, - &NetAddress::OnionV2(_) => { 12 }, - &NetAddress::OnionV3 { .. } => { 37 }, + &SocketAddress::TcpIpV4 { .. } => { 6 }, + &SocketAddress::TcpIpV6 { .. } => { 18 }, + &SocketAddress::OnionV2(_) => { 12 }, + &SocketAddress::OnionV3 { .. } => { 37 }, // Consists of 1-byte hostname length, hostname bytes, and 2-byte port. - &NetAddress::Hostname { ref hostname, .. } => { u16::from(hostname.len()) + 3 }, + &SocketAddress::Hostname { ref hostname, .. } => { u16::from(hostname.len()) + 3 }, } } @@ -822,31 +822,31 @@ impl NetAddress { pub(crate) const MAX_LEN: u16 = 258; } -impl Writeable for NetAddress { +impl Writeable for SocketAddress { fn write(&self, writer: &mut W) -> Result<(), io::Error> { match self { - &NetAddress::IPv4 { ref addr, ref port } => { + &SocketAddress::TcpIpV4 { ref addr, ref port } => { 1u8.write(writer)?; addr.write(writer)?; port.write(writer)?; }, - &NetAddress::IPv6 { ref addr, ref port } => { + &SocketAddress::TcpIpV6 { ref addr, ref port } => { 2u8.write(writer)?; addr.write(writer)?; port.write(writer)?; }, - &NetAddress::OnionV2(bytes) => { + &SocketAddress::OnionV2(bytes) => { 3u8.write(writer)?; bytes.write(writer)?; }, - &NetAddress::OnionV3 { ref ed25519_pubkey, ref checksum, ref version, ref port } => { + &SocketAddress::OnionV3 { ref ed25519_pubkey, ref checksum, ref version, ref port } => { 4u8.write(writer)?; ed25519_pubkey.write(writer)?; checksum.write(writer)?; version.write(writer)?; port.write(writer)?; }, - &NetAddress::Hostname { ref hostname, ref port } => { + &SocketAddress::Hostname { ref hostname, ref port } => { 5u8.write(writer)?; hostname.write(writer)?; port.write(writer)?; @@ -856,25 +856,25 @@ impl Writeable for NetAddress { } } -impl Readable for Result { - fn read(reader: &mut R) -> Result, DecodeError> { +impl Readable for Result { + fn read(reader: &mut R) -> Result, DecodeError> { let byte = ::read(reader)?; match byte { 1 => { - Ok(Ok(NetAddress::IPv4 { + Ok(Ok(SocketAddress::TcpIpV4 { addr: Readable::read(reader)?, port: Readable::read(reader)?, })) }, 2 => { - Ok(Ok(NetAddress::IPv6 { + Ok(Ok(SocketAddress::TcpIpV6 { addr: Readable::read(reader)?, port: Readable::read(reader)?, })) }, - 3 => Ok(Ok(NetAddress::OnionV2(Readable::read(reader)?))), + 3 => Ok(Ok(SocketAddress::OnionV2(Readable::read(reader)?))), 4 => { - Ok(Ok(NetAddress::OnionV3 { + Ok(Ok(SocketAddress::OnionV3 { ed25519_pubkey: Readable::read(reader)?, checksum: Readable::read(reader)?, version: Readable::read(reader)?, @@ -882,7 +882,7 @@ impl Readable for Result { })) }, 5 => { - Ok(Ok(NetAddress::Hostname { + Ok(Ok(SocketAddress::Hostname { hostname: Readable::read(reader)?, port: Readable::read(reader)?, })) @@ -892,8 +892,8 @@ impl Readable for Result { } } -impl Readable for NetAddress { - fn read(reader: &mut R) -> Result { +impl Readable for SocketAddress { + fn read(reader: &mut R) -> Result { match Readable::read(reader) { Ok(Ok(res)) => Ok(res), Ok(Err(_)) => Err(DecodeError::UnknownVersion), @@ -902,9 +902,9 @@ impl Readable for NetAddress { } } -/// [`NetAddress`] error variants +/// [`SocketAddress`] error variants #[derive(Debug, Eq, PartialEq, Clone)] -pub enum NetAddressParseError { +pub enum SocketAddressParseError { /// Socket address (IPv4/IPv6) parsing error SocketAddrParse, /// Invalid input format @@ -915,34 +915,34 @@ pub enum NetAddressParseError { InvalidOnionV3, } -impl fmt::Display for NetAddressParseError { +impl fmt::Display for SocketAddressParseError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - NetAddressParseError::SocketAddrParse => write!(f, "Socket address (IPv4/IPv6) parsing error"), - NetAddressParseError::InvalidInput => write!(f, "Invalid input format. \ + SocketAddressParseError::SocketAddrParse => write!(f, "Socket address (IPv4/IPv6) parsing error"), + SocketAddressParseError::InvalidInput => write!(f, "Invalid input format. \ Expected: \":\", \"[]:\", \".onion:\" or \":\""), - NetAddressParseError::InvalidPort => write!(f, "Invalid port"), - NetAddressParseError::InvalidOnionV3 => write!(f, "Invalid onion v3 address"), + SocketAddressParseError::InvalidPort => write!(f, "Invalid port"), + SocketAddressParseError::InvalidOnionV3 => write!(f, "Invalid onion v3 address"), } } } #[cfg(feature = "std")] -impl From for NetAddress { +impl From for SocketAddress { fn from(addr: std::net::SocketAddrV4) -> Self { - NetAddress::IPv4 { addr: addr.ip().octets(), port: addr.port() } + SocketAddress::TcpIpV4 { addr: addr.ip().octets(), port: addr.port() } } } #[cfg(feature = "std")] -impl From for NetAddress { +impl From for SocketAddress { fn from(addr: std::net::SocketAddrV6) -> Self { - NetAddress::IPv6 { addr: addr.ip().octets(), port: addr.port() } + SocketAddress::TcpIpV6 { addr: addr.ip().octets(), port: addr.port() } } } #[cfg(feature = "std")] -impl From for NetAddress { +impl From for SocketAddress { fn from(addr: std::net::SocketAddr) -> Self { match addr { std::net::SocketAddr::V4(addr) => addr.into(), @@ -951,15 +951,15 @@ impl From for NetAddress { } } -fn parse_onion_address(host: &str, port: u16) -> Result { +fn parse_onion_address(host: &str, port: u16) -> Result { if host.ends_with(".onion") { let domain = &host[..host.len() - ".onion".len()]; if domain.len() != 56 { - return Err(NetAddressParseError::InvalidOnionV3); + return Err(SocketAddressParseError::InvalidOnionV3); } - let onion = base32::Alphabet::RFC4648 { padding: false }.decode(&domain).map_err(|_| NetAddressParseError::InvalidOnionV3)?; + let onion = base32::Alphabet::RFC4648 { padding: false }.decode(&domain).map_err(|_| SocketAddressParseError::InvalidOnionV3)?; if onion.len() != 35 { - return Err(NetAddressParseError::InvalidOnionV3); + return Err(SocketAddressParseError::InvalidOnionV3); } let version = onion[0]; let first_checksum_flag = onion[1]; @@ -967,16 +967,16 @@ fn parse_onion_address(host: &str, port: u16) -> Result Result { match std::net::SocketAddr::from_str(s) { @@ -984,17 +984,17 @@ impl FromStr for NetAddress { Err(_) => { let trimmed_input = match s.rfind(":") { Some(pos) => pos, - None => return Err(NetAddressParseError::InvalidInput), + None => return Err(SocketAddressParseError::InvalidInput), }; let host = &s[..trimmed_input]; - let port: u16 = s[trimmed_input + 1..].parse().map_err(|_| NetAddressParseError::InvalidPort)?; + let port: u16 = s[trimmed_input + 1..].parse().map_err(|_| SocketAddressParseError::InvalidPort)?; if host.ends_with(".onion") { return parse_onion_address(host, port); }; if let Ok(hostname) = Hostname::try_from(s[..trimmed_input].to_string()) { - return Ok(NetAddress::Hostname { hostname, port }); + return Ok(SocketAddress::Hostname { hostname, port }); }; - return Err(NetAddressParseError::SocketAddrParse) + return Err(SocketAddressParseError::SocketAddrParse) }, } } @@ -1039,7 +1039,7 @@ pub struct UnsignedNodeAnnouncement { /// This should be sanitized before use. There is no guarantee of uniqueness. pub alias: NodeAlias, /// List of addresses on which this node is reachable - pub addresses: Vec, + pub addresses: Vec, pub(crate) excess_address_data: Vec, pub(crate) excess_data: Vec, } @@ -1874,7 +1874,7 @@ impl Readable for Init { fn read(r: &mut R) -> Result { let global_features: InitFeatures = Readable::read(r)?; let features: InitFeatures = Readable::read(r)?; - let mut remote_network_address: Option = None; + let mut remote_network_address: Option = None; let mut networks: Option>> = None; decode_tlv_stream!(r, { (1, networks, option), @@ -2373,7 +2373,7 @@ impl Readable for UnsignedNodeAnnouncement { let alias: NodeAlias = Readable::read(r)?; let addr_len: u16 = Readable::read(r)?; - let mut addresses: Vec = Vec::new(); + let mut addresses: Vec = Vec::new(); let mut addr_readpos = 0; let mut excess = false; let mut excess_byte = 0; @@ -2580,7 +2580,7 @@ mod tests { use crate::ln::ChannelId; use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures}; use crate::ln::msgs::{self, FinalOnionHopData, OnionErrorPacket}; - use crate::ln::msgs::NetAddress; + use crate::ln::msgs::SocketAddress; use crate::routing::gossip::{NodeAlias, NodeId}; use crate::util::ser::{Writeable, Readable, Hostname, TransactionU16LenLimited}; @@ -2601,7 +2601,7 @@ mod tests { #[cfg(feature = "std")] use std::net::{Ipv4Addr, Ipv6Addr}; - use crate::ln::msgs::NetAddressParseError; + use crate::ln::msgs::SocketAddressParseError; #[test] fn encoding_channel_reestablish() { @@ -2768,24 +2768,24 @@ mod tests { }; let mut addresses = Vec::new(); if ipv4 { - addresses.push(NetAddress::IPv4 { + addresses.push(SocketAddress::TcpIpV4 { addr: [255, 254, 253, 252], port: 9735 }); } if ipv6 { - addresses.push(NetAddress::IPv6 { + addresses.push(SocketAddress::TcpIpV6 { addr: [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240], port: 9735 }); } if onionv2 { - addresses.push(NetAddress::OnionV2( + addresses.push(msgs::SocketAddress::OnionV2( [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 38, 7] )); } if onionv3 { - addresses.push(NetAddress::OnionV3 { + addresses.push(msgs::SocketAddress::OnionV3 { ed25519_pubkey: [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, 228, 227, 226, 225, 224], checksum: 32, version: 16, @@ -2793,7 +2793,7 @@ mod tests { }); } if hostname { - addresses.push(NetAddress::Hostname { + addresses.push(SocketAddress::Hostname { hostname: Hostname::try_from(String::from("host")).unwrap(), port: 9735, }); @@ -3609,7 +3609,7 @@ mod tests { }.encode(), hex::decode("00000000014001010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202").unwrap()); let init_msg = msgs::Init { features: InitFeatures::from_le_bytes(vec![]), networks: Some(vec![mainnet_hash]), - remote_network_address: Some(NetAddress::IPv4 { + remote_network_address: Some(SocketAddress::TcpIpV4 { addr: [127, 0, 0, 1], port: 1000, }), @@ -3977,44 +3977,44 @@ mod tests { #[test] #[cfg(feature = "std")] - fn test_net_address_from_str() { - assert_eq!(NetAddress::IPv4 { + fn test_socket_address_from_str() { + assert_eq!(SocketAddress::TcpIpV4 { addr: Ipv4Addr::new(127, 0, 0, 1).octets(), port: 1234, - }, NetAddress::from_str("127.0.0.1:1234").unwrap()); + }, SocketAddress::from_str("127.0.0.1:1234").unwrap()); - assert_eq!(NetAddress::IPv6 { + assert_eq!(SocketAddress::TcpIpV6 { addr: Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1).octets(), port: 1234, - }, NetAddress::from_str("[0:0:0:0:0:0:0:1]:1234").unwrap()); + }, SocketAddress::from_str("[0:0:0:0:0:0:0:1]:1234").unwrap()); assert_eq!( - NetAddress::Hostname { + SocketAddress::Hostname { hostname: Hostname::try_from("lightning-node.mydomain.com".to_string()).unwrap(), port: 1234, - }, NetAddress::from_str("lightning-node.mydomain.com:1234").unwrap()); + }, SocketAddress::from_str("lightning-node.mydomain.com:1234").unwrap()); assert_eq!( - NetAddress::Hostname { + SocketAddress::Hostname { hostname: Hostname::try_from("example.com".to_string()).unwrap(), port: 1234, - }, NetAddress::from_str("example.com:1234").unwrap()); - assert_eq!(NetAddress::OnionV3 { + }, SocketAddress::from_str("example.com:1234").unwrap()); + assert_eq!(SocketAddress::OnionV3 { ed25519_pubkey: [37, 24, 75, 5, 25, 73, 117, 194, 139, 102, 182, 107, 4, 105, 247, 246, 85, 111, 177, 172, 49, 137, 167, 155, 64, 221, 163, 47, 31, 33, 71, 3], checksum: 48326, version: 121, port: 1234 - }, NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:1234").unwrap()); - assert_eq!(Err(NetAddressParseError::InvalidOnionV3), NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6.onion:1234")); - assert_eq!(Err(NetAddressParseError::InvalidInput), NetAddress::from_str("127.0.0.1@1234")); - assert_eq!(Err(NetAddressParseError::InvalidInput), "".parse::()); - assert!(NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:9735:94").is_err()); - assert!(NetAddress::from_str("wrong$%#.com:1234").is_err()); - assert_eq!(Err(NetAddressParseError::InvalidPort), NetAddress::from_str("example.com:wrong")); - assert!("localhost".parse::().is_err()); - assert!("localhost:invalid-port".parse::().is_err()); - assert!( "invalid-onion-v3-hostname.onion:8080".parse::().is_err()); - assert!("b32.example.onion:invalid-port".parse::().is_err()); - assert!("invalid-address".parse::().is_err()); - assert!(NetAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:1234").is_err()); + }, SocketAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:1234").unwrap()); + assert_eq!(Err(SocketAddressParseError::InvalidOnionV3), SocketAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6.onion:1234")); + assert_eq!(Err(SocketAddressParseError::InvalidInput), SocketAddress::from_str("127.0.0.1@1234")); + assert_eq!(Err(SocketAddressParseError::InvalidInput), "".parse::()); + assert!(SocketAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:9735:94").is_err()); + assert!(SocketAddress::from_str("wrong$%#.com:1234").is_err()); + assert_eq!(Err(SocketAddressParseError::InvalidPort), SocketAddress::from_str("example.com:wrong")); + assert!("localhost".parse::().is_err()); + assert!("localhost:invalid-port".parse::().is_err()); + assert!( "invalid-onion-v3-hostname.onion:8080".parse::().is_err()); + assert!("b32.example.onion:invalid-port".parse::().is_err()); + assert!("invalid-address".parse::().is_err()); + assert!(SocketAddress::from_str("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion.onion:1234").is_err()); } } diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index ddf51cdd796..0cc9e7e0531 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -18,6 +18,7 @@ use crate::events::{self, PaymentFailureReason}; use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; use crate::ln::channelmanager::{ChannelDetails, EventCompletionAction, HTLCSource, PaymentId}; use crate::ln::onion_utils::{DecodedOnionFailure, HTLCFailReason}; +use crate::offers::invoice::Bolt12Invoice; use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteParameters, Router}; use crate::util::errors::APIError; use crate::util::logger::Logger; @@ -52,9 +53,11 @@ pub(crate) enum PendingOutboundPayment { }, AwaitingInvoice { timer_ticks_without_response: u8, + retry_strategy: Retry, }, InvoiceReceived { payment_hash: PaymentHash, + retry_strategy: Retry, }, Retryable { retry_strategy: Option, @@ -155,7 +158,7 @@ impl PendingOutboundPayment { match self { PendingOutboundPayment::Legacy { .. } => None, PendingOutboundPayment::AwaitingInvoice { .. } => None, - PendingOutboundPayment::InvoiceReceived { payment_hash } => Some(*payment_hash), + PendingOutboundPayment::InvoiceReceived { payment_hash, .. } => Some(*payment_hash), PendingOutboundPayment::Retryable { payment_hash, .. } => Some(*payment_hash), PendingOutboundPayment::Fulfilled { payment_hash, .. } => *payment_hash, PendingOutboundPayment::Abandoned { payment_hash, .. } => Some(*payment_hash), @@ -185,7 +188,7 @@ impl PendingOutboundPayment { payment_hash: *payment_hash, reason: Some(reason) }; - } else if let PendingOutboundPayment::InvoiceReceived { payment_hash } = self { + } else if let PendingOutboundPayment::InvoiceReceived { payment_hash, .. } = self { *self = PendingOutboundPayment::Abandoned { session_privs: HashSet::new(), payment_hash: *payment_hash, @@ -262,7 +265,7 @@ pub enum Retry { /// Each attempt may be multiple HTLCs along multiple paths if the router decides to split up a /// retry, and may retry multiple failed HTLCs at once if they failed around the same time and /// were retried along a route from a single call to [`Router::find_route_with_id`]. - Attempts(usize), + Attempts(u32), #[cfg(not(feature = "no-std"))] /// Time elapsed before abandoning retries for a payment. At least one attempt at payment is made; /// see [`PaymentParameters::expiry_time`] to avoid any attempt at payment after a specific time. @@ -271,6 +274,19 @@ pub enum Retry { Timeout(core::time::Duration), } +#[cfg(feature = "no-std")] +impl_writeable_tlv_based_enum!(Retry, + ; + (0, Attempts) +); + +#[cfg(not(feature = "no-std"))] +impl_writeable_tlv_based_enum!(Retry, + ; + (0, Attempts), + (2, Timeout) +); + impl Retry { pub(crate) fn is_retryable_now(&self, attempts: &PaymentAttempts) -> bool { match (self, attempts) { @@ -304,7 +320,7 @@ pub(crate) type PaymentAttempts = PaymentAttemptsUsingTime; pub(crate) struct PaymentAttemptsUsingTime { /// This count will be incremented only after the result of the attempt is known. When it's 0, /// it means the result of the first attempt is not known yet. - pub(crate) count: usize, + pub(crate) count: u32, /// This field is only used when retry is `Retry::Timeout` which is only build with feature std #[cfg(not(feature = "no-std"))] first_attempted_at: T, @@ -440,6 +456,15 @@ pub enum PaymentSendFailure { }, } +/// An error when attempting to pay a BOLT 12 invoice. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum Bolt12PaymentError { + /// The invoice was not requested. + UnexpectedInvoice, + /// Payment for an invoice with the corresponding [`PaymentId`] was already initiated. + DuplicateInvoice, +} + /// Information which is provided, encrypted, to the payment recipient when sending HTLCs. /// /// This should generally be constructed with data communicated to us from the recipient (via a @@ -677,6 +702,50 @@ impl OutboundPayments { } } + #[allow(unused)] + pub(super) fn send_payment_for_bolt12_invoice( + &self, invoice: &Bolt12Invoice, payment_id: PaymentId, router: &R, + first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, node_signer: &NS, + best_block_height: u32, logger: &L, + pending_events: &Mutex)>>, + send_payment_along_path: SP, + ) -> Result<(), Bolt12PaymentError> + where + R::Target: Router, + ES::Target: EntropySource, + NS::Target: NodeSigner, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(SendAlongPathArgs) -> Result<(), APIError>, + { + let payment_hash = invoice.payment_hash(); + match self.pending_outbound_payments.lock().unwrap().entry(payment_id) { + hash_map::Entry::Occupied(entry) => match entry.get() { + PendingOutboundPayment::AwaitingInvoice { retry_strategy, .. } => { + *entry.into_mut() = PendingOutboundPayment::InvoiceReceived { + payment_hash, + retry_strategy: *retry_strategy, + }; + }, + _ => return Err(Bolt12PaymentError::DuplicateInvoice), + }, + hash_map::Entry::Vacant(_) => return Err(Bolt12PaymentError::UnexpectedInvoice), + }; + + let route_params = RouteParameters { + payment_params: PaymentParameters::from_bolt12_invoice(&invoice), + final_value_msat: invoice.amount_msats(), + }; + + self.find_route_and_send_payment( + payment_hash, payment_id, route_params, router, first_hops, &inflight_htlcs, + entropy_source, node_signer, best_block_height, logger, pending_events, + &send_payment_along_path + ); + + Ok(()) + } + pub(super) fn check_retry_payments( &self, router: &R, first_hops: FH, inflight_htlcs: IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, @@ -711,7 +780,7 @@ impl OutboundPayments { } core::mem::drop(outbounds); if let Some((payment_hash, payment_id, route_params)) = retry_id_route_params { - self.retry_payment_internal(payment_hash, payment_id, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) + self.find_route_and_send_payment(payment_hash, payment_id, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) } else { break } } @@ -797,7 +866,7 @@ impl OutboundPayments { Ok(()) } - fn retry_payment_internal( + fn find_route_and_send_payment( &self, payment_hash: PaymentHash, payment_id: PaymentId, route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, @@ -902,12 +971,26 @@ impl OutboundPayments { log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); return }, - PendingOutboundPayment::AwaitingInvoice { .. } | - PendingOutboundPayment::InvoiceReceived { .. } => - { + PendingOutboundPayment::AwaitingInvoice { .. } => { log_error!(logger, "Payment not yet sent"); return }, + PendingOutboundPayment::InvoiceReceived { payment_hash, retry_strategy } => { + let total_amount = route_params.final_value_msat; + let recipient_onion = RecipientOnionFields { + payment_secret: None, + payment_metadata: None, + custom_tlvs: vec![], + }; + let retry_strategy = Some(*retry_strategy); + let payment_params = Some(route_params.payment_params.clone()); + let (retryable_payment, onion_session_privs) = self.create_pending_payment( + *payment_hash, recipient_onion.clone(), None, &route, + retry_strategy, payment_params, entropy_source, best_block_height + ); + *payment.into_mut() = retryable_payment; + (total_amount, recipient_onion, None, onion_session_privs) + }, PendingOutboundPayment::Fulfilled { .. } => { log_error!(logger, "Payment already completed"); return @@ -950,14 +1033,14 @@ impl OutboundPayments { match err { PaymentSendFailure::AllFailedResendSafe(errs) => { Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, errs.into_iter().map(|e| Err(e)), logger, pending_events); - self.retry_payment_internal(payment_hash, payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + self.find_route_and_send_payment(payment_hash, payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, PaymentSendFailure::PartialFailure { failed_paths_retry: Some(mut retry), results, .. } => { Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut retry, route.paths, results.into_iter(), logger, pending_events); // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. - self.retry_payment_internal(payment_hash, payment_id, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + self.find_route_and_send_payment(payment_hash, payment_id, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. } => { // This may happen if we send a payment and some paths fail, but only due to a temporary @@ -1070,48 +1153,67 @@ impl OutboundPayments { keysend_preimage: Option, route: &Route, retry_strategy: Option, payment_params: Option, entropy_source: &ES, best_block_height: u32 ) -> Result, PaymentSendFailure> where ES::Target: EntropySource { - let mut onion_session_privs = Vec::with_capacity(route.paths.len()); - for _ in 0..route.paths.len() { - onion_session_privs.push(entropy_source.get_secure_random_bytes()); - } - let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap(); match pending_outbounds.entry(payment_id) { hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::DuplicatePayment), hash_map::Entry::Vacant(entry) => { - let payment = entry.insert(PendingOutboundPayment::Retryable { - retry_strategy, - attempts: PaymentAttempts::new(), - payment_params, - session_privs: HashSet::new(), - pending_amt_msat: 0, - pending_fee_msat: Some(0), - payment_hash, - payment_secret: recipient_onion.payment_secret, - payment_metadata: recipient_onion.payment_metadata, - keysend_preimage, - custom_tlvs: recipient_onion.custom_tlvs, - starting_block_height: best_block_height, - total_msat: route.get_total_amount(), - }); - - for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { - assert!(payment.insert(*session_priv_bytes, path)); - } - + let (payment, onion_session_privs) = self.create_pending_payment( + payment_hash, recipient_onion, keysend_preimage, route, retry_strategy, + payment_params, entropy_source, best_block_height + ); + entry.insert(payment); Ok(onion_session_privs) }, } } + fn create_pending_payment( + &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, + keysend_preimage: Option, route: &Route, retry_strategy: Option, + payment_params: Option, entropy_source: &ES, best_block_height: u32 + ) -> (PendingOutboundPayment, Vec<[u8; 32]>) + where + ES::Target: EntropySource, + { + let mut onion_session_privs = Vec::with_capacity(route.paths.len()); + for _ in 0..route.paths.len() { + onion_session_privs.push(entropy_source.get_secure_random_bytes()); + } + + let mut payment = PendingOutboundPayment::Retryable { + retry_strategy, + attempts: PaymentAttempts::new(), + payment_params, + session_privs: HashSet::new(), + pending_amt_msat: 0, + pending_fee_msat: Some(0), + payment_hash, + payment_secret: recipient_onion.payment_secret, + payment_metadata: recipient_onion.payment_metadata, + keysend_preimage, + custom_tlvs: recipient_onion.custom_tlvs, + starting_block_height: best_block_height, + total_msat: route.get_total_amount(), + }; + + for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { + assert!(payment.insert(*session_priv_bytes, path)); + } + + (payment, onion_session_privs) + } + #[allow(unused)] - pub(super) fn add_new_awaiting_invoice(&self, payment_id: PaymentId) -> Result<(), ()> { + pub(super) fn add_new_awaiting_invoice( + &self, payment_id: PaymentId, retry_strategy: Retry + ) -> Result<(), ()> { let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap(); match pending_outbounds.entry(payment_id) { hash_map::Entry::Occupied(_) => Err(()), hash_map::Entry::Vacant(entry) => { entry.insert(PendingOutboundPayment::AwaitingInvoice { timer_ticks_without_response: 0, + retry_strategy, }); Ok(()) @@ -1586,9 +1688,11 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment, }, (5, AwaitingInvoice) => { (0, timer_ticks_without_response, required), + (2, retry_strategy, required), }, (7, InvoiceReceived) => { (0, payment_hash, required), + (2, retry_strategy, required), }, ); @@ -1602,7 +1706,10 @@ mod tests { use crate::ln::channelmanager::{PaymentId, RecipientOnionFields}; use crate::ln::features::{ChannelFeatures, NodeFeatures}; use crate::ln::msgs::{ErrorAction, LightningError}; - use crate::ln::outbound_payment::{INVOICE_REQUEST_TIMEOUT_TICKS, OutboundPayments, Retry, RetryableSendFailure}; + use crate::ln::outbound_payment::{Bolt12PaymentError, INVOICE_REQUEST_TIMEOUT_TICKS, OutboundPayments, Retry, RetryableSendFailure}; + use crate::offers::invoice::DEFAULT_RELATIVE_EXPIRY; + use crate::offers::offer::OfferBuilder; + use crate::offers::test_utils::*; use crate::routing::gossip::NetworkGraph; use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteHop, RouteParameters}; use crate::sync::{Arc, Mutex, RwLock}; @@ -1661,7 +1768,7 @@ mod tests { PaymentId([0; 32]), None, &Route { paths: vec![], route_params: None }, Some(Retry::Attempts(1)), Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); - outbound_payments.retry_payment_internal( + outbound_payments.find_route_and_send_payment( PaymentHash([0; 32]), PaymentId([0; 32]), expired_route_params, &&router, vec![], &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_| Ok(())); @@ -1705,7 +1812,7 @@ mod tests { PaymentId([0; 32]), None, &Route { paths: vec![], route_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); - outbound_payments.retry_payment_internal( + outbound_payments.find_route_and_send_payment( PaymentHash([0; 32]), PaymentId([0; 32]), route_params, &&router, vec![], &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_| Ok(())); @@ -1807,7 +1914,7 @@ mod tests { let payment_id = PaymentId([0; 32]); assert!(!outbound_payments.has_pending_payments()); - assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); assert!(outbound_payments.has_pending_payments()); for _ in 0..INVOICE_REQUEST_TIMEOUT_TICKS { @@ -1825,10 +1932,10 @@ mod tests { ); assert!(pending_events.lock().unwrap().is_empty()); - assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); assert!(outbound_payments.has_pending_payments()); - assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_err()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_err()); } #[test] @@ -1838,7 +1945,7 @@ mod tests { let payment_id = PaymentId([0; 32]); assert!(!outbound_payments.has_pending_payments()); - assert!(outbound_payments.add_new_awaiting_invoice(payment_id).is_ok()); + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); assert!(outbound_payments.has_pending_payments()); outbound_payments.abandon_payment( @@ -1852,4 +1959,240 @@ mod tests { ); assert!(pending_events.lock().unwrap().is_empty()); } + + #[cfg(feature = "std")] + #[test] + fn fails_sending_payment_for_expired_bolt12_invoice() { + let logger = test_utils::TestLogger::new(); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = RwLock::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); + let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); + + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + let created_at = now() - DEFAULT_RELATIVE_EXPIRY; + let invoice = OfferBuilder::new("foo".into(), recipient_pubkey()) + .amount_msats(1000) + .build().unwrap() + .request_invoice(vec![1; 32], payer_pubkey()).unwrap() + .build().unwrap() + .sign(payer_sign).unwrap() + .respond_with_no_std(payment_paths(), payment_hash(), created_at).unwrap() + .build().unwrap() + .sign(recipient_sign).unwrap(); + + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| panic!() + ), + Ok(()), + ); + assert!(!outbound_payments.has_pending_payments()); + + let payment_hash = invoice.payment_hash(); + let reason = Some(PaymentFailureReason::PaymentExpired); + + assert!(!pending_events.lock().unwrap().is_empty()); + assert_eq!( + pending_events.lock().unwrap().pop_front(), + Some((Event::PaymentFailed { payment_id, payment_hash, reason }, None)), + ); + assert!(pending_events.lock().unwrap().is_empty()); + } + + #[test] + fn fails_finding_route_for_bolt12_invoice() { + let logger = test_utils::TestLogger::new(); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = RwLock::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); + let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); + + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + let invoice = OfferBuilder::new("foo".into(), recipient_pubkey()) + .amount_msats(1000) + .build().unwrap() + .request_invoice(vec![1; 32], payer_pubkey()).unwrap() + .build().unwrap() + .sign(payer_sign).unwrap() + .respond_with_no_std(payment_paths(), payment_hash(), now()).unwrap() + .build().unwrap() + .sign(recipient_sign).unwrap(); + + router.expect_find_route( + RouteParameters { + payment_params: PaymentParameters::from_bolt12_invoice(&invoice), + final_value_msat: invoice.amount_msats(), + }, + Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError }), + ); + + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| panic!() + ), + Ok(()), + ); + assert!(!outbound_payments.has_pending_payments()); + + let payment_hash = invoice.payment_hash(); + let reason = Some(PaymentFailureReason::RouteNotFound); + + assert!(!pending_events.lock().unwrap().is_empty()); + assert_eq!( + pending_events.lock().unwrap().pop_front(), + Some((Event::PaymentFailed { payment_id, payment_hash, reason }, None)), + ); + assert!(pending_events.lock().unwrap().is_empty()); + } + + #[test] + fn fails_paying_for_bolt12_invoice() { + let logger = test_utils::TestLogger::new(); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = RwLock::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); + let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); + + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + let invoice = OfferBuilder::new("foo".into(), recipient_pubkey()) + .amount_msats(1000) + .build().unwrap() + .request_invoice(vec![1; 32], payer_pubkey()).unwrap() + .build().unwrap() + .sign(payer_sign).unwrap() + .respond_with_no_std(payment_paths(), payment_hash(), now()).unwrap() + .build().unwrap() + .sign(recipient_sign).unwrap(); + + let route_params = RouteParameters { + payment_params: PaymentParameters::from_bolt12_invoice(&invoice), + final_value_msat: invoice.amount_msats(), + }; + router.expect_find_route( + route_params.clone(), Ok(Route { paths: vec![], route_params: Some(route_params) }) + ); + + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| panic!() + ), + Ok(()), + ); + assert!(!outbound_payments.has_pending_payments()); + + let payment_hash = invoice.payment_hash(); + let reason = Some(PaymentFailureReason::UnexpectedError); + + assert!(!pending_events.lock().unwrap().is_empty()); + assert_eq!( + pending_events.lock().unwrap().pop_front(), + Some((Event::PaymentFailed { payment_id, payment_hash, reason }, None)), + ); + assert!(pending_events.lock().unwrap().is_empty()); + } + + #[test] + fn sends_payment_for_bolt12_invoice() { + let logger = test_utils::TestLogger::new(); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = RwLock::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); + let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); + + let pending_events = Mutex::new(VecDeque::new()); + let outbound_payments = OutboundPayments::new(); + let payment_id = PaymentId([0; 32]); + + let invoice = OfferBuilder::new("foo".into(), recipient_pubkey()) + .amount_msats(1000) + .build().unwrap() + .request_invoice(vec![1; 32], payer_pubkey()).unwrap() + .build().unwrap() + .sign(payer_sign).unwrap() + .respond_with_no_std(payment_paths(), payment_hash(), now()).unwrap() + .build().unwrap() + .sign(recipient_sign).unwrap(); + + let route_params = RouteParameters { + payment_params: PaymentParameters::from_bolt12_invoice(&invoice), + final_value_msat: invoice.amount_msats(), + }; + router.expect_find_route( + route_params.clone(), + Ok(Route { + paths: vec![ + Path { + hops: vec![ + RouteHop { + pubkey: recipient_pubkey(), + node_features: NodeFeatures::empty(), + short_channel_id: 42, + channel_features: ChannelFeatures::empty(), + fee_msat: invoice.amount_msats(), + cltv_expiry_delta: 0, + } + ], + blinded_tail: None, + } + ], + route_params: Some(route_params), + }) + ); + + assert!(!outbound_payments.has_pending_payments()); + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| panic!() + ), + Err(Bolt12PaymentError::UnexpectedInvoice), + ); + assert!(!outbound_payments.has_pending_payments()); + assert!(pending_events.lock().unwrap().is_empty()); + + assert!(outbound_payments.add_new_awaiting_invoice(payment_id, Retry::Attempts(0)).is_ok()); + assert!(outbound_payments.has_pending_payments()); + + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| Ok(()) + ), + Ok(()), + ); + assert!(outbound_payments.has_pending_payments()); + assert!(pending_events.lock().unwrap().is_empty()); + + assert_eq!( + outbound_payments.send_payment_for_bolt12_invoice( + &invoice, payment_id, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, + &&keys_manager, 0, &&logger, &pending_events, |_| panic!() + ), + Err(Bolt12PaymentError::DuplicateInvoice), + ); + assert!(outbound_payments.has_pending_payments()); + assert!(pending_events.lock().unwrap().is_empty()); + } } diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 7565246fe34..1e0294073a2 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -23,7 +23,7 @@ use crate::events::{MessageSendEvent, MessageSendEventsProvider, OnionMessagePro use crate::ln::ChannelId; use crate::ln::features::{InitFeatures, NodeFeatures}; use crate::ln::msgs; -use crate::ln::msgs::{ChannelMessageHandler, LightningError, NetAddress, OnionMessageHandler, RoutingMessageHandler}; +use crate::ln::msgs::{ChannelMessageHandler, LightningError, SocketAddress, OnionMessageHandler, RoutingMessageHandler}; use crate::ln::channelmanager::{SimpleArcChannelManager, SimpleRefChannelManager}; use crate::util::ser::{VecWriter, Writeable, Writer}; use crate::ln::peer_channel_encryptor::{PeerChannelEncryptor,NextNoiseStep}; @@ -483,7 +483,7 @@ struct Peer { /// handshake and can talk to this peer normally (though use [`Peer::handshake_complete`] to /// check this. their_features: Option, - their_net_address: Option, + their_socket_address: Option, pending_outbound_buffer: LinkedList>, pending_outbound_buffer_first_msg_offset: usize, @@ -855,28 +855,28 @@ impl core::fmt::Display for OptionalFromDebugger<'_> { /// A function used to filter out local or private addresses /// /// -fn filter_addresses(ip_address: Option) -> Option { +fn filter_addresses(ip_address: Option) -> Option { match ip_address{ // For IPv4 range 10.0.0.0 - 10.255.255.255 (10/8) - Some(NetAddress::IPv4{addr: [10, _, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [10, _, _, _], port: _}) => None, // For IPv4 range 0.0.0.0 - 0.255.255.255 (0/8) - Some(NetAddress::IPv4{addr: [0, _, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [0, _, _, _], port: _}) => None, // For IPv4 range 100.64.0.0 - 100.127.255.255 (100.64/10) - Some(NetAddress::IPv4{addr: [100, 64..=127, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [100, 64..=127, _, _], port: _}) => None, // For IPv4 range 127.0.0.0 - 127.255.255.255 (127/8) - Some(NetAddress::IPv4{addr: [127, _, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [127, _, _, _], port: _}) => None, // For IPv4 range 169.254.0.0 - 169.254.255.255 (169.254/16) - Some(NetAddress::IPv4{addr: [169, 254, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [169, 254, _, _], port: _}) => None, // For IPv4 range 172.16.0.0 - 172.31.255.255 (172.16/12) - Some(NetAddress::IPv4{addr: [172, 16..=31, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [172, 16..=31, _, _], port: _}) => None, // For IPv4 range 192.168.0.0 - 192.168.255.255 (192.168/16) - Some(NetAddress::IPv4{addr: [192, 168, _, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [192, 168, _, _], port: _}) => None, // For IPv4 range 192.88.99.0 - 192.88.99.255 (192.88.99/24) - Some(NetAddress::IPv4{addr: [192, 88, 99, _], port: _}) => None, + Some(SocketAddress::TcpIpV4{addr: [192, 88, 99, _], port: _}) => None, // For IPv6 range 2000:0000:0000:0000:0000:0000:0000:0000 - 3fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff (2000::/3) - Some(NetAddress::IPv6{addr: [0x20..=0x3F, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], port: _}) => ip_address, + Some(SocketAddress::TcpIpV6{addr: [0x20..=0x3F, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], port: _}) => ip_address, // For remaining addresses - Some(NetAddress::IPv6{addr: _, port: _}) => None, + Some(SocketAddress::TcpIpV6{addr: _, port: _}) => None, Some(..) => ip_address, None => None, } @@ -933,14 +933,14 @@ impl Vec<(PublicKey, Option)> { + pub fn get_peer_node_ids(&self) -> Vec<(PublicKey, Option)> { let peers = self.peers.read().unwrap(); peers.values().filter_map(|peer_mutex| { let p = peer_mutex.lock().unwrap(); if !p.handshake_complete() { return None; } - Some((p.their_node_id.unwrap().0, p.their_net_address.clone())) + Some((p.their_node_id.unwrap().0, p.their_socket_address.clone())) }).collect() } @@ -973,7 +973,7 @@ impl) -> Result, PeerHandleError> { + pub fn new_outbound_connection(&self, their_node_id: PublicKey, descriptor: Descriptor, remote_network_address: Option) -> Result, PeerHandleError> { let mut peer_encryptor = PeerChannelEncryptor::new_outbound(their_node_id.clone(), self.get_ephemeral_key()); let res = peer_encryptor.get_act_one(&self.secp_ctx).to_vec(); let pending_read_buffer = [0; 50].to_vec(); // Noise act two is 50 bytes @@ -989,7 +989,7 @@ impl) -> Result<(), PeerHandleError> { + pub fn new_inbound_connection(&self, descriptor: Descriptor, remote_network_address: Option) -> Result<(), PeerHandleError> { let peer_encryptor = PeerChannelEncryptor::new_inbound(&self.node_signer); let pending_read_buffer = [0; 50].to_vec(); // Noise act one is 50 bytes @@ -1045,7 +1045,7 @@ impl) { + pub fn broadcast_node_announcement(&self, rgb: [u8; 3], alias: [u8; 32], mut addresses: Vec) { if addresses.len() > 100 { panic!("More than half the message size was taken up by public addresses!"); } @@ -2488,7 +2488,7 @@ mod tests { use crate::ln::peer_channel_encryptor::PeerChannelEncryptor; use crate::ln::peer_handler::{CustomMessageHandler, PeerManager, MessageHandler, SocketDescriptor, IgnoringMessageHandler, filter_addresses}; use crate::ln::{msgs, wire}; - use crate::ln::msgs::{LightningError, NetAddress}; + use crate::ln::msgs::{LightningError, SocketAddress}; use crate::util::test_utils; use bitcoin::Network; @@ -2647,13 +2647,13 @@ mod tests { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_a = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1000}; + let addr_a = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1000}; let id_b = peer_b.node_signer.get_node_id(Recipient::Node).unwrap(); let mut fd_b = FileDescriptor { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_b = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1001}; + let addr_b = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1001}; let initial_data = peer_b.new_outbound_connection(id_a, fd_b.clone(), Some(addr_a.clone())).unwrap(); peer_a.new_inbound_connection(fd_a.clone(), Some(addr_b.clone())).unwrap(); assert_eq!(peer_a.read_event(&mut fd_a, &initial_data).unwrap(), false); @@ -2698,12 +2698,12 @@ mod tests { fd: $id + ctr * 3, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_a = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1000}; + let addr_a = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1000}; let mut fd_b = FileDescriptor { fd: $id + ctr * 3, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_b = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1001}; + let addr_b = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1001}; let initial_data = peers[1].new_outbound_connection(id_a, fd_b.clone(), Some(addr_a.clone())).unwrap(); peers[0].new_inbound_connection(fd_a.clone(), Some(addr_b.clone())).unwrap(); if peers[0].read_event(&mut fd_a, &initial_data).is_err() { break; } @@ -2770,12 +2770,12 @@ mod tests { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_a = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1000}; + let addr_a = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1000}; let mut fd_b = FileDescriptor { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_b = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1001}; + let addr_b = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1001}; let initial_data = peer_b.new_outbound_connection(id_a, fd_b.clone(), Some(addr_a.clone())).unwrap(); peer_a.new_inbound_connection(fd_a.clone(), Some(addr_b.clone())).unwrap(); assert_eq!(peer_a.read_event(&mut fd_a, &initial_data).unwrap(), false); @@ -2806,12 +2806,12 @@ mod tests { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_a = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1000}; + let addr_a = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1000}; let mut fd_b = FileDescriptor { fd: 1, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_b = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1001}; + let addr_b = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1001}; let initial_data = peer_b.new_outbound_connection(id_a, fd_b.clone(), Some(addr_a.clone())).unwrap(); peer_a.new_inbound_connection(fd_a.clone(), Some(addr_b.clone())).unwrap(); assert_eq!(peer_a.read_event(&mut fd_a, &initial_data).unwrap(), false); @@ -2888,7 +2888,7 @@ mod tests { fd: 3, outbound_data: Arc::new(Mutex::new(Vec::new())), disconnect: Arc::new(AtomicBool::new(false)), }; - let addr_dup = NetAddress::IPv4{addr: [127, 0, 0, 1], port: 1003}; + let addr_dup = SocketAddress::TcpIpV4{addr: [127, 0, 0, 1], port: 1003}; let id_a = cfgs[0].node_signer.get_node_id(Recipient::Node).unwrap(); peers[0].new_inbound_connection(fd_dup.clone(), Some(addr_dup.clone())).unwrap(); @@ -3026,91 +3026,91 @@ mod tests { // Tests the filter_addresses function. // For (10/8) - let ip_address = NetAddress::IPv4{addr: [10, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [10, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [10, 0, 255, 201], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [10, 0, 255, 201], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [10, 255, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [10, 255, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (0/8) - let ip_address = NetAddress::IPv4{addr: [0, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [0, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [0, 0, 255, 187], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [0, 0, 255, 187], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [0, 255, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [0, 255, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (100.64/10) - let ip_address = NetAddress::IPv4{addr: [100, 64, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [100, 64, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [100, 78, 255, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [100, 78, 255, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [100, 127, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [100, 127, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (127/8) - let ip_address = NetAddress::IPv4{addr: [127, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [127, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [127, 65, 73, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [127, 65, 73, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [127, 255, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [127, 255, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (169.254/16) - let ip_address = NetAddress::IPv4{addr: [169, 254, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [169, 254, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [169, 254, 221, 101], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [169, 254, 221, 101], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [169, 254, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [169, 254, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (172.16/12) - let ip_address = NetAddress::IPv4{addr: [172, 16, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [172, 16, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [172, 27, 101, 23], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [172, 27, 101, 23], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [172, 31, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [172, 31, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (192.168/16) - let ip_address = NetAddress::IPv4{addr: [192, 168, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 168, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [192, 168, 205, 159], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 168, 205, 159], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [192, 168, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 168, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (192.88.99/24) - let ip_address = NetAddress::IPv4{addr: [192, 88, 99, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 88, 99, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [192, 88, 99, 140], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 88, 99, 140], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv4{addr: [192, 88, 99, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [192, 88, 99, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For other IPv4 addresses - let ip_address = NetAddress::IPv4{addr: [188, 255, 99, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [188, 255, 99, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); - let ip_address = NetAddress::IPv4{addr: [123, 8, 129, 14], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [123, 8, 129, 14], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); - let ip_address = NetAddress::IPv4{addr: [2, 88, 9, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV4{addr: [2, 88, 9, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); // For (2000::/3) - let ip_address = NetAddress::IPv6{addr: [32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); - let ip_address = NetAddress::IPv6{addr: [45, 34, 209, 190, 0, 123, 55, 34, 0, 0, 3, 27, 201, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [45, 34, 209, 190, 0, 123, 55, 34, 0, 0, 3, 27, 201, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); - let ip_address = NetAddress::IPv6{addr: [63, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [63, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), Some(ip_address.clone())); // For other IPv6 addresses - let ip_address = NetAddress::IPv6{addr: [24, 240, 12, 32, 0, 0, 0, 0, 20, 97, 0, 32, 121, 254, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [24, 240, 12, 32, 0, 0, 0, 0, 20, 97, 0, 32, 121, 254, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv6{addr: [68, 23, 56, 63, 0, 0, 2, 7, 75, 109, 0, 39, 0, 0, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [68, 23, 56, 63, 0, 0, 2, 7, 75, 109, 0, 39, 0, 0, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); - let ip_address = NetAddress::IPv6{addr: [101, 38, 140, 230, 100, 0, 30, 98, 0, 26, 0, 0, 57, 96, 0, 0], port: 1000}; + let ip_address = SocketAddress::TcpIpV6{addr: [101, 38, 140, 230, 100, 0, 30, 98, 0, 26, 0, 0, 57, 96, 0, 0], port: 1000}; assert_eq!(filter_addresses(Some(ip_address.clone())), None); // For (None) diff --git a/lightning/src/offers/invoice.rs b/lightning/src/offers/invoice.rs index 06215e2d486..908d2d4bee6 100644 --- a/lightning/src/offers/invoice.rs +++ b/lightning/src/offers/invoice.rs @@ -129,7 +129,7 @@ use crate::prelude::*; #[cfg(feature = "std")] use std::time::SystemTime; -const DEFAULT_RELATIVE_EXPIRY: Duration = Duration::from_secs(7200); +pub(crate) const DEFAULT_RELATIVE_EXPIRY: Duration = Duration::from_secs(7200); /// Tag for the hash function used when signing a [`Bolt12Invoice`]'s merkle root. pub const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature"); diff --git a/lightning/src/offers/mod.rs b/lightning/src/offers/mod.rs index c6883abca34..3593b14f1a8 100644 --- a/lightning/src/offers/mod.rs +++ b/lightning/src/offers/mod.rs @@ -24,4 +24,4 @@ mod payer; pub mod refund; pub(crate) mod signer; #[cfg(test)] -mod test_utils; +pub(crate) mod test_utils; diff --git a/lightning/src/offers/test_utils.rs b/lightning/src/offers/test_utils.rs index f1b3c79edc0..39122472eac 100644 --- a/lightning/src/offers/test_utils.rs +++ b/lightning/src/offers/test_utils.rs @@ -20,33 +20,33 @@ use crate::ln::features::BlindedHopFeatures; use crate::offers::invoice::BlindedPayInfo; use crate::offers::merkle::TaggedHash; -pub(super) fn payer_keys() -> KeyPair { +pub(crate) fn payer_keys() -> KeyPair { let secp_ctx = Secp256k1::new(); KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()) } -pub(super) fn payer_sign>(message: &T) -> Result { +pub(crate) fn payer_sign>(message: &T) -> Result { let secp_ctx = Secp256k1::new(); let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()); Ok(secp_ctx.sign_schnorr_no_aux_rand(message.as_ref().as_digest(), &keys)) } -pub(super) fn payer_pubkey() -> PublicKey { +pub(crate) fn payer_pubkey() -> PublicKey { payer_keys().public_key() } -pub(super) fn recipient_keys() -> KeyPair { +pub(crate) fn recipient_keys() -> KeyPair { let secp_ctx = Secp256k1::new(); KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()) } -pub(super) fn recipient_sign>(message: &T) -> Result { +pub(crate) fn recipient_sign>(message: &T) -> Result { let secp_ctx = Secp256k1::new(); let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()); Ok(secp_ctx.sign_schnorr_no_aux_rand(message.as_ref().as_digest(), &keys)) } -pub(super) fn recipient_pubkey() -> PublicKey { +pub(crate) fn recipient_pubkey() -> PublicKey { recipient_keys().public_key() } @@ -59,7 +59,7 @@ pub(super) fn privkey(byte: u8) -> SecretKey { SecretKey::from_slice(&[byte; 32]).unwrap() } -pub(super) fn payment_paths() -> Vec<(BlindedPayInfo, BlindedPath)> { +pub(crate) fn payment_paths() -> Vec<(BlindedPayInfo, BlindedPath)> { let paths = vec![ BlindedPath { introduction_node_id: pubkey(40), @@ -101,17 +101,17 @@ pub(super) fn payment_paths() -> Vec<(BlindedPayInfo, BlindedPath)> { payinfo.into_iter().zip(paths.into_iter()).collect() } -pub(super) fn payment_hash() -> PaymentHash { +pub(crate) fn payment_hash() -> PaymentHash { PaymentHash([42; 32]) } -pub(super) fn now() -> Duration { +pub(crate) fn now() -> Duration { std::time::SystemTime::now() .duration_since(std::time::SystemTime::UNIX_EPOCH) .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH") } -pub(super) struct FixedEntropy; +pub(crate) struct FixedEntropy; impl EntropySource for FixedEntropy { fn get_secure_random_bytes(&self) -> [u8; 32] { diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 64f3585b953..ef776a44dc1 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -25,7 +25,7 @@ use bitcoin::blockdata::constants::genesis_block; use crate::events::{MessageSendEvent, MessageSendEventsProvider}; use crate::ln::ChannelId; use crate::ln::features::{ChannelFeatures, NodeFeatures, InitFeatures}; -use crate::ln::msgs::{DecodeError, ErrorAction, Init, LightningError, RoutingMessageHandler, NetAddress, MAX_VALUE_MSAT}; +use crate::ln::msgs::{DecodeError, ErrorAction, Init, LightningError, RoutingMessageHandler, SocketAddress, MAX_VALUE_MSAT}; use crate::ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, GossipTimestampFilter}; use crate::ln::msgs::{QueryChannelRange, ReplyChannelRange, QueryShortChannelIds, ReplyShortChannelIdsEnd}; use crate::ln::msgs; @@ -1128,7 +1128,7 @@ pub struct NodeAnnouncementInfo { impl NodeAnnouncementInfo { /// Internet-level addresses via which one can connect to the node - pub fn addresses(&self) -> &[NetAddress] { + pub fn addresses(&self) -> &[SocketAddress] { self.announcement_message.as_ref() .map(|msg| msg.contents.addresses.as_slice()) .unwrap_or_default() @@ -1137,7 +1137,7 @@ impl NodeAnnouncementInfo { impl Writeable for NodeAnnouncementInfo { fn write(&self, writer: &mut W) -> Result<(), io::Error> { - let empty_addresses = Vec::::new(); + let empty_addresses = Vec::::new(); write_tlv_fields!(writer, { (0, self.features, required), (2, self.last_update, required), @@ -1160,7 +1160,7 @@ impl Readable for NodeAnnouncementInfo { (8, announcement_message, option), (10, _addresses, optional_vec), // deprecated, not used anymore }); - let _: Option> = _addresses; + let _: Option> = _addresses; Ok(Self { features: features.0.unwrap(), last_update: last_update.0.unwrap(), rgb: rgb.0.unwrap(), alias: alias.0.unwrap(), announcement_message }) } @@ -1236,7 +1236,7 @@ impl Writeable for NodeInfo { } // A wrapper allowing for the optional deserialization of `NodeAnnouncementInfo`. Utilizing this is -// necessary to maintain compatibility with previous serializations of `NetAddress` that have an +// necessary to maintain compatibility with previous serializations of `SocketAddress` that have an // invalid hostname set. We ignore and eat all errors until we are either able to read a // `NodeAnnouncementInfo` or hit a `ShortRead`, i.e., read the TLV field to the end. struct NodeAnnouncementInfoDeserWrapper(NodeAnnouncementInfo); @@ -2039,7 +2039,7 @@ impl ReadOnlyNetworkGraph<'_> { /// Get network addresses by node id. /// Returns None if the requested node is completely unknown, /// or if node announcement for the node was never received. - pub fn get_addresses(&self, pubkey: &PublicKey) -> Option> { + pub fn get_addresses(&self, pubkey: &PublicKey) -> Option> { self.nodes.get(&NodeId::from_pubkey(&pubkey)) .and_then(|node| node.announcement_info.as_ref().map(|ann| ann.addresses().to_vec())) } diff --git a/pending_changelog/move_netaddress_to_socketaddress.txt b/pending_changelog/move_netaddress_to_socketaddress.txt new file mode 100644 index 00000000000..5153ed1d035 --- /dev/null +++ b/pending_changelog/move_netaddress_to_socketaddress.txt @@ -0,0 +1 @@ +* The `NetAddress` has been moved to `SocketAddress`. The fieds `IPv4` and `IPv6` are also rename to `TcpIpV4` and `TcpIpV6` (#2358).