Merge branch 'network-dependent-grpc-ports' into merge-dev-into-featu…

…re-dan

* network-dependent-grpc-ports:
  feat: different default grpc ports for different networks
  fix(clients): fix tari nodejs client proto paths (tari-project#4743)
  chore: disallow onion v2 (tari-project#4745)
  feat: change priority in mempool to take into account age (tari-project#4737)
  feat: trigger mempool sync on lag (tari-project#4730)
  fix(core): use compact inputs for block propagation (tari-project#4714)
  ci: deny dbg macro (tari-project#4740)

.circleci/config.yml

@@ -12,7 +12,7 @@ commands:
           command: node -v
       - run:
           name: npm ci
-          command: cd integration_tests && npm ci
+          command: cd clients/nodejs/base_node_grpc_client && npm install && cd ../wallet_grpc_client && npm install && cd ../../../integration_tests && npm install
       - run:
           name: Check formatting
           command: cd integration_tests && npm run check-fmt
@@ -54,7 +54,7 @@ commands:
           command: node -v
       - run:
           name: npm ci
-          command: cd integration_tests && npm ci
+          command: cd clients/nodejs/base_node_grpc_client && npm install && cd ../wallet_grpc_client && npm install && cd ../../../integration_tests && npm install
       - run:
           name: Check eslint
           command: cd integration_tests && npm run lint

.github/workflows/integration_tests.yml

@@ -120,7 +120,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: lts/erbium
           cache: 'npm'
           cache-dependency-path: integration_tests/package-lock.json
 

.gitignore

@@ -60,4 +60,5 @@ buildtools/Output/
 
 clients/base_node_grpc_client/package-lock.json
 clients/validator_node_grpc_client/package-lock.json
-clients/wallet_grpc_client/package-lock.json
+clients/wallet_grpc_client/package-lock.json
+pie/

applications/tari_app_grpc/src/conversions/transaction_input.rs

@@ -114,10 +114,7 @@ impl TryFrom<TransactionInput> for grpc::TransactionInput {
                     .commitment()
                     .map_err(|_| "Non-compact Transaction input should contain commitment".to_string())?
                     .to_vec(),
-                hash: input
-                    .canonical_hash()
-                    .map_err(|_| "Non-compact Transaction input should be able to be hashed".to_string())?
-                    .to_vec(),
+                hash: input.canonical_hash().to_vec(),
 
                 script: input
                     .script()

applications/tari_base_node/src/commands/command/check_db.rs

@@ -54,7 +54,7 @@ impl CommandContext {
             io::stdout().flush().await?;
             // we can only check till the pruning horizon, 0 is archive node so it needs to check every block.
             if height > horizon_height {
-                match self.node_service.get_block(height).await {
+                match self.node_service.get_block(height, false).await {
                     Err(err) => {
                         // We need to check the data itself, as FetchMatchingBlocks will suppress any error, only
                         // logging it.

applications/tari_base_node/src/commands/command/get_block.rs

@@ -70,7 +70,7 @@ impl CommandContext {
     pub async fn get_block(&self, height: u64, format: Format) -> Result<(), Error> {
         let block = self
             .blockchain_db
-            .fetch_blocks(height..=height)
+            .fetch_blocks(height..=height, false)
             .await?
             .pop()
             .ok_or(ArgsError::NotFoundAt { height })?;
@@ -90,7 +90,7 @@ impl CommandContext {
     pub async fn get_block_by_hash(&self, hash: HashOutput, format: Format) -> Result<(), Error> {
         let block = self
             .blockchain_db
-            .fetch_block_by_hash(hash)
+            .fetch_block_by_hash(hash, false)
             .await?
             .ok_or(ArgsError::NotFound)?;
         match format {

applications/tari_base_node/src/commands/command/period_stats.rs

@@ -75,13 +75,13 @@ impl CommandContext {
 
             let block = self
                 .node_service
-                .get_block(height)
+                .get_block(height, true)
                 .await?
                 .ok_or_else(|| anyhow!("Error in db, block not found at height {}", height))?;
 
             let prev_block = self
                 .node_service
-                .get_block(height - 1)
+                .get_block(height - 1, true)
                 .await?
                 .ok_or_else(|| anyhow!("Error in db, block not found at height {}", height))?;
 

applications/tari_base_node/src/config.rs

@@ -144,7 +144,7 @@ impl Default for BaseNodeConfig {
         Self {
             override_from: None,
             network: Network::default(),
-            grpc_address: Some("/ip4/127.0.0.1/tcp/18142".parse().unwrap()),
+            grpc_address: None,
             identity_file: PathBuf::from("config/base_node_id.json"),
             use_libtor: false,
             tor_identity_file: PathBuf::from("config/base_node_tor_id.json"),

applications/tari_base_node/src/grpc/base_node_grpc_server.rs

@@ -386,7 +386,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
             for (start, end) in page_iter {
                 debug!(target: LOG_TARGET, "Page: {}-{}", start, end);
                 // TODO: Better error handling
-                let result_data = match handler.get_blocks(start..=end).await {
+                let result_data = match handler.get_blocks(start..=end, true).await {
                     Err(err) => {
                         warn!(target: LOG_TARGET, "Internal base node service error: {}", err);
                         return;
@@ -852,7 +852,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
         task::spawn(async move {
             let page_iter = NonOverlappingIntegerPairIter::new(start, end + 1, GET_BLOCKS_PAGE_SIZE);
             for (start, end) in page_iter {
-                let blocks = match handler.get_blocks(start..=end).await {
+                let blocks = match handler.get_blocks(start..=end, false).await {
                     Err(err) => {
                         warn!(
                             target: LOG_TARGET,
@@ -1628,7 +1628,7 @@ async fn get_block_group(
 
     let (start, end) = get_heights(&height_request, handler.clone()).await?;
 
-    let blocks = match handler.get_blocks(start..=end).await {
+    let blocks = match handler.get_blocks(start..=end, false).await {
         Err(err) => {
             warn!(
                 target: LOG_TARGET,

applications/tari_base_node/src/main.rs

@@ -94,7 +94,10 @@ use log::*;
 use opentelemetry::{self, global, KeyValue};
 use tari_app_utilities::{consts, identity_management::setup_node_identity, utilities::setup_runtime};
 use tari_common::{
-    configuration::{bootstrap::ApplicationType, Network},
+    configuration::{
+        bootstrap::{grpc_default_port, ApplicationType},
+        Network,
+    },
     exit_codes::{ExitCode, ExitError},
     initialize_logging,
     load_configuration,
@@ -229,11 +232,13 @@ async fn run_node(
     // Build, node, build!
     let ctx = builder::configure_and_initialize_node(config.clone(), node_identity, shutdown.to_signal()).await?;
 
-    if let Some(address) = config.base_node.grpc_address.clone() {
-        // Go, GRPC, go go
-        let grpc = crate::grpc::base_node_grpc_server::BaseNodeGrpcServer::from_base_node_context(&ctx);
-        task::spawn(run_grpc(grpc, address, shutdown.to_signal()));
-    }
+    let grpc_address = config.base_node.grpc_address.clone().unwrap_or_else(|| {
+        let port = grpc_default_port(ApplicationType::BaseNode, config.base_node.network);
+        format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap()
+    });
+    // Go, GRPC, go go
+    let grpc = grpc::base_node_grpc_server::BaseNodeGrpcServer::from_base_node_context(&ctx);
+    task::spawn(run_grpc(grpc, grpc_address, shutdown.to_signal()));
 
     // Run, node, run!
     let context = CommandContext::new(&ctx, shutdown);
@@ -288,7 +293,7 @@ fn enable_tracing() {
 
 /// Runs the gRPC server
 async fn run_grpc(
-    grpc: crate::grpc::base_node_grpc_server::BaseNodeGrpcServer,
+    grpc: grpc::base_node_grpc_server::BaseNodeGrpcServer,
     grpc_address: Multiaddr,
     interrupt_signal: ShutdownSignal,
 ) -> Result<(), anyhow::Error> {

applications/tari_base_node/src/recovery.rs

@@ -154,7 +154,7 @@ async fn do_recovery<D: BlockchainBackend + 'static>(
         io::stdout().flush().unwrap();
         trace!(target: LOG_TARGET, "Asking for block with height: {}", counter);
         let block = source_database
-            .fetch_block(counter)
+            .fetch_block(counter, true)
             .map_err(|e| anyhow!("Could not get block from recovery db: {}", e))?
             .try_into_block()?;
         trace!(target: LOG_TARGET, "Adding block: {}", block);

applications/tari_console_wallet/src/init/mod.rs

@@ -27,7 +27,10 @@ use rpassword::prompt_password_stdout;
 use rustyline::Editor;
 use tari_app_utilities::identity_management::setup_node_identity;
 use tari_common::{
-    configuration::bootstrap::prompt,
+    configuration::{
+        bootstrap::{grpc_default_port, prompt, ApplicationType},
+        Network,
+    },
     exit_codes::{ExitCode, ExitError},
 };
 use tari_comms::{
@@ -41,7 +44,7 @@ use tari_crypto::keys::PublicKey;
 use tari_key_manager::{cipher_seed::CipherSeed, mnemonic::MnemonicLanguage};
 use tari_p2p::{peer_seeds::SeedPeer, TransportType};
 use tari_shutdown::ShutdownSignal;
-use tari_utilities::{ByteArray, SafePassword};
+use tari_utilities::{hex::Hex, ByteArray, SafePassword};
 use tari_wallet::{
     error::{WalletError, WalletStorageError},
     output_manager_service::storage::database::OutputManagerDatabase,
@@ -164,7 +167,7 @@ pub async fn get_base_node_peer_config(
 
     // If the user has not explicitly set a base node in the config, we try detect one
     if !non_interactive_mode && config.wallet.custom_base_node.is_none() {
-        if let Some(detected_node) = detect_local_base_node().await {
+        if let Some(detected_node) = detect_local_base_node(config.wallet.network).await {
             match selected_base_node {
                 Some(ref base_node) if base_node.public_key == detected_node.public_key => {
                     // Skip asking because it's already set
@@ -407,15 +410,31 @@ pub async fn init_wallet(
     Ok(wallet)
 }
 
-async fn detect_local_base_node() -> Option<SeedPeer> {
+async fn detect_local_base_node(network: Network) -> Option<SeedPeer> {
     use tari_app_grpc::tari_rpc::{base_node_client::BaseNodeClient, Empty};
-    const COMMON_BASE_NODE_GRPC_ADDRESS: &str = "http://127.0.0.1:18142";
+    let addr = format!(
+        "http://127.0.0.1:{}",
+        grpc_default_port(ApplicationType::BaseNode, network)
+    );
+    debug!(target: LOG_TARGET, "Checking for local base node at {}", addr);
 
-    let mut node_conn = BaseNodeClient::connect(COMMON_BASE_NODE_GRPC_ADDRESS).await.ok()?;
+    let mut node_conn = match BaseNodeClient::connect(addr).await.ok() {
+        Some(conn) => conn,
+        None => {
+            debug!(target: LOG_TARGET, "No local base node detected");
+            return None;
+        },
+    };
     let resp = node_conn.identify(Empty {}).await.ok()?;
     let identity = resp.get_ref();
     let public_key = CommsPublicKey::from_bytes(&identity.public_key).ok()?;
     let address = Multiaddr::from_str(&identity.public_address).ok()?;
+    debug!(
+        target: LOG_TARGET,
+        "Local base node found with pk={} and addr={}",
+        public_key.to_hex(),
+        address
+    );
     Some(SeedPeer::new(public_key, vec![address]))
 }
 

applications/tari_console_wallet/src/main.rs

@@ -38,7 +38,7 @@ use opentelemetry::{self, global, KeyValue};
 use recovery::prompt_private_key_from_seed_words;
 use tari_app_utilities::consts;
 use tari_common::{
-    configuration::bootstrap::ApplicationType,
+    configuration::bootstrap::{grpc_default_port, ApplicationType},
     exit_codes::{ExitCode, ExitError},
     initialize_logging,
     load_configuration,
@@ -99,9 +99,10 @@ fn main_inner() -> Result<(), ExitError> {
         include_str!("../log4rs_sample.yml"),
     )?;
 
-    #[cfg_attr(not(all(unix, feature = "libtor")), allow(unused_mut))]
     let mut config = ApplicationConfig::load_from(&cfg)?;
 
+    setup_grpc_config(&mut config);
+
     let runtime = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
@@ -270,3 +271,16 @@ fn enable_tracing() {
     tracing::subscriber::set_global_default(subscriber)
         .expect("Tracing could not be set. Try running without `--tracing-enabled`");
 }
+
+fn setup_grpc_config(config: &mut ApplicationConfig) {
+    if config.wallet.grpc_enabled && config.wallet.grpc_address.is_none() {
+        config.wallet.grpc_address = Some(
+            format!(
+                "/ip4/127.0.0.1/tcp/{}",
+                grpc_default_port(ApplicationType::ConsoleWallet, config.wallet.network)
+            )
+            .parse()
+            .unwrap(),
+        );
+    }
+}

applications/tari_console_wallet/src/wallet_modes.rs

@@ -267,12 +267,10 @@ pub fn tui_mode(
 ) -> Result<(), ExitError> {
     let (events_broadcaster, _events_listener) = broadcast::channel(100);
     if config.grpc_enabled {
-        let grpc = WalletGrpcServer::new(wallet.clone());
-        handle.spawn(run_grpc(
-            grpc,
-            config.grpc_address.clone(),
-            config.grpc_authentication.clone(),
-        ));
+        if let Some(address) = config.grpc_address.clone() {
+            let grpc = WalletGrpcServer::new(wallet.clone());
+            handle.spawn(run_grpc(grpc, address, config.grpc_authentication.clone()));
+        }
     }
 
     let notifier = Notifier::new(
@@ -369,11 +367,11 @@ pub fn recovery_mode(
 
 pub fn grpc_mode(handle: Handle, config: &WalletConfig, wallet: WalletSqlite) -> Result<(), ExitError> {
     info!(target: LOG_TARGET, "Starting grpc server");
-    if config.grpc_enabled {
+    if let Some(address) = config.grpc_address.as_ref().filter(|_| config.grpc_enabled).cloned() {
         let grpc = WalletGrpcServer::new(wallet);
         let auth = config.grpc_authentication.clone();
         handle
-            .block_on(run_grpc(grpc, config.grpc_address.clone(), auth))
+            .block_on(run_grpc(grpc, address, auth))
             .map_err(|e| ExitError::new(ExitCode::GrpcError, e))?;
     } else {
         println!("GRPC server is disabled");

base_layer/core/src/base_node/comms_interface/comms_request.rs

@@ -46,20 +46,38 @@ pub enum NodeCommsRequest {
     FetchHeaders(RangeInclusive<u64>),
     FetchHeadersByHashes(Vec<HashOutput>),
     FetchMatchingUtxos(Vec<HashOutput>),
-    FetchMatchingBlocks(RangeInclusive<u64>),
-    FetchBlocksByHash(Vec<HashOutput>),
+    FetchMatchingBlocks {
+        range: RangeInclusive<u64>,
+        compact: bool,
+    },
+    FetchBlocksByHash {
+        block_hashes: Vec<HashOutput>,
+        compact: bool,
+    },
     FetchBlocksByKernelExcessSigs(Vec<Signature>),
     FetchBlocksByUtxos(Vec<Commitment>),
     GetHeaderByHash(HashOutput),
     GetBlockByHash(HashOutput),
     GetNewBlockTemplate(GetNewBlockTemplateRequest),
     GetNewBlock(NewBlockTemplate),
     FetchKernelByExcessSig(Signature),
-    FetchMempoolTransactionsByExcessSigs { excess_sigs: Vec<PrivateKey> },
-    FetchValidatorNodesKeys { height: u64 },
-    FetchCommittee { height: u64, shard: [u8; 32] },
-    GetShardKey { height: u64, public_key: PublicKey },
-    FetchTemplateRegistrations { from_height: u64 },
+    FetchMempoolTransactionsByExcessSigs {
+        excess_sigs: Vec<PrivateKey>,
+    },
+    FetchValidatorNodesKeys {
+        height: u64,
+    },
+    FetchCommittee {
+        height: u64,
+        shard: [u8; 32],
+    },
+    GetShardKey {
+        height: u64,
+        public_key: PublicKey,
+    },
+    FetchTemplateRegistrations {
+        from_height: u64,
+    },
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -79,10 +97,12 @@ impl Display for NodeCommsRequest {
             },
             FetchHeadersByHashes(v) => write!(f, "FetchHeadersByHashes (n={})", v.len()),
             FetchMatchingUtxos(v) => write!(f, "FetchMatchingUtxos (n={})", v.len()),
-            FetchMatchingBlocks(range) => {
-                write!(f, "FetchMatchingBlocks ({:?})", range)
+            FetchMatchingBlocks { range, compact } => {
+                write!(f, "FetchMatchingBlocks ({:?}, {})", range, compact)
+            },
+            FetchBlocksByHash { block_hashes, compact } => {
+                write!(f, "FetchBlocksByHash (n={}, {})", block_hashes.len(), compact)
             },
-            FetchBlocksByHash(v) => write!(f, "FetchBlocksByHash (n={})", v.len()),
             FetchBlocksByKernelExcessSigs(v) => write!(f, "FetchBlocksByKernelExcessSigs (n={})", v.len()),
             FetchBlocksByUtxos(v) => write!(f, "FetchBlocksByUtxos (n={})", v.len()),
             GetHeaderByHash(v) => write!(f, "GetHeaderByHash({})", v.to_hex()),

base_layer/core/src/base_node/comms_interface/error.rs

@@ -20,6 +20,7 @@
 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+use tari_common_types::types::FixedHash;
 use tari_comms_dht::outbound::DhtOutboundError;
 use tari_service_framework::reply_channel::TransportChannelError;
 use thiserror::Error;
@@ -67,4 +68,6 @@ pub enum CommsInterfaceError {
     BlockError(#[from] BlockError),
     #[error("Invalid request for {request}: {details}")]
     InvalidRequest { request: &'static str, details: String },
+    #[error("Peer sent invalid full block {hash}: {details}")]
+    InvalidFullBlock { hash: FixedHash, details: String },
 }