feat(comms)!: add signature to peer identity to allow third party ide…

…ntity updates (#3629)

Description
---

- add signature to peers with update timestamp to allow updates
- update peer sync, identity, discovery and join protocols to validate signature (backward compatible)
- signature is optional, if not provided the peer addresses is not updated
- add migration for peer db
- remove old migrations
- add some unit tests
- include PR #3624 
- fix: use of `from_timestamp` on unvalidated data can cause a panic, replaced with `from_timestamp_opt`
- decouple comms cipher key from wallet and implement identity signing after loading (👁️ 🙏 @philipr-za )
- add list connected public keys to FFI (👁️ 🙏  @StriderDM)
- update cucumber tests as needed

Compatibility
PeerDB: All changes are forward compatible, once upgraded the previous peer db cannot be used.
DhtProtocol: fully compatible, identity signature is optional and will have the same effect as the current protocol if not provided
Identity Protocol: fully compatible, identity signature is optional but SHOULD be provided once upgraded
NodeIdentity json file: will be updated on node startup

Motivation and Context
---
Allow address changes to be securely propagated through the network by third party peers

How Has This Been Tested?
---
Some basic unit tests
Compatibility: manually with two upgraded base nodes connecting to existing weatherwax nodes, console wallet tested

applications/ffi_client/lib/index.js

@@ -17,6 +17,7 @@ const {
   u8ArrayPtr,
   byteVectorRef,
   publicKeyRef,
+  publicKeyArrPtr,
   strArray,
   strArrayPtr,
 } = require("./types");
@@ -30,7 +31,9 @@ const libWallet = ffi.Library("./libtari_wallet_ffi.dylib", {
     commsConfigRef,
     ["string", transportRef, "string", "string", u64, u64, errPtr],
   ],
+  comms_list_connected_public_keys: [publicKeyArrPtr, [walletRef, errPtr]],
   public_key_create: [publicKeyRef, [byteVectorRef, errPtr]],
+  public_keys_destroy: ["void", [publicKeyArrPtr]],
   public_key_get_bytes: [u8ArrayPtr, [publicKeyRef, errPtr]],
   seed_words_create: [strPtr, []],
   seed_words_get_at: ["string", [strArrayPtr, u32, errPtr]],

applications/ffi_client/lib/types.js

@@ -17,7 +17,8 @@ const u8Array = ArrayType(ref.types.uint8);
 const u8ArrayPtr = ref.refType(u8Array);
 const byteVectorRef = ref.refType(u8Array);
 const publicKeyRef = ref.refType(ref.types.void);
-const strArray = ArrayType("string");
+const publicKeyArrPtr = ref.refType(u8Array);
+const strArray = ref.refType(ArrayType(ref.types.void));
 const strArrayPtr = ref.refType(ArrayType("string"));
 
 module.exports = {
@@ -37,6 +38,7 @@ module.exports = {
   u8ArrayPtr,
   byteVectorRef,
   publicKeyRef,
+  publicKeyArrPtr,
   strArray,
   strArrayPtr,
 };

applications/launchpad/docker_rig/config.toml

@@ -232,7 +232,7 @@ console_wallet_tor_identity_file = "config/console_wallet_tor.json"
 # direcly over TCP. /ip4, /ip6, /dns, /dns4 and /dns6 are supported.
 # tor_proxy_bypass_addresses = ["/dns4/my-foo-base-node/tcp/9998"]
 # When using the tor transport and set to true, outbound TCP connections bypass the tor proxy. Defaults to false for better privacy
-# tor_proxy_bypass_for_outbound_tcp = false;
+# tor_proxy_bypass_for_outbound_tcp = false
 
 ########################################################################################################################
 #                                                                                                                      #
@@ -309,7 +309,7 @@ console_wallet_tor_identity_file = "config/igor/console_wallet_tor.json"
 # direcly over TCP. /ip4, /ip6, /dns, /dns4 and /dns6 are supported.
 # tor_proxy_bypass_addresses = ["/dns4/my-foo-base-node/tcp/9998"]
 # When using the tor transport and set to true, outbound TCP connections bypass the tor proxy. Defaults to false for better privacy
-# tor_proxy_bypass_for_outbound_tcp = false;
+# tor_proxy_bypass_for_outbound_tcp = false
 
 ########################################################################################################################
 #                                                                                                                      #

applications/tari_app_utilities/Cargo.toml

@@ -16,11 +16,11 @@ config = { version = "0.9.3" }
 futures = { version = "^0.3.16", default-features = false, features = ["alloc"] }
 qrcode = { version = "0.12" }
 dirs-next = "1.0.2"
-serde = "1.0.126"
 json5 = "0.2.2"
 log = { version = "0.4.8", features = ["std"] }
 rand = "0.8"
 tokio = { version = "1.11", features = ["signal"] }
+serde = "1.0.126"
 structopt = { version = "0.3.13", default_features = false }
 strum = "^0.22"
 strum_macros = "^0.22"

applications/tari_app_utilities/src/identity_management.rs

@@ -29,9 +29,8 @@ use tari_common::{
     configuration::{bootstrap::prompt, utils::get_local_ip},
     exit_codes::ExitCodes,
 };
-use tari_common_types::types::PrivateKey;
 use tari_comms::{multiaddr::Multiaddr, peer_manager::PeerFeatures, NodeIdentity};
-use tari_crypto::{keys::SecretKey, tari_utilities::hex::Hex};
+use tari_crypto::tari_utilities::hex::Hex;
 
 pub const LOG_TARGET: &str = "tari_application";
 
@@ -59,7 +58,7 @@ pub fn setup_node_identity<P: AsRef<Path>>(
             None => Ok(Arc::new(id)),
         },
         Err(e) => {
-            debug!(target: LOG_TARGET, "Node id not found. {}. Creating new ID", e);
+            debug!(target: LOG_TARGET, "Failed to load node identity: {}", e);
             if !create_id {
                 let prompt = prompt("Node identity does not exist.\nWould you like to to create one (Y/n)?");
                 if !prompt {
@@ -79,6 +78,8 @@ pub fn setup_node_identity<P: AsRef<Path>>(
                 };
             }
 
+            debug!(target: LOG_TARGET, "Existing node id not found. {}. Creating new ID", e);
+
             match create_new_identity(&identity_file, public_address.clone(), peer_features) {
                 Ok(id) => {
                     info!(
@@ -117,7 +118,7 @@ pub fn load_identity<P: AsRef<Path>>(path: P) -> Result<NodeIdentity, String> {
         ));
     }
 
-    let id_str = std::fs::read_to_string(path.as_ref()).map_err(|e| {
+    let id_str = fs::read_to_string(path.as_ref()).map_err(|e| {
         format!(
             "The node identity file, {}, could not be read. {}",
             path.as_ref().to_str().unwrap_or("?"),
@@ -131,6 +132,10 @@ pub fn load_identity<P: AsRef<Path>>(path: P) -> Result<NodeIdentity, String> {
             e
         )
     })?;
+    // Check whether the previous version has a signature and sign if necessary
+    if !id.is_signed() {
+        id.sign();
+    }
     debug!(
         "Node ID loaded with public key {} and Node id {}",
         id.public_key().to_hex(),
@@ -151,9 +156,8 @@ pub fn create_new_identity<P: AsRef<Path>>(
     public_addr: Option<Multiaddr>,
     features: PeerFeatures,
 ) -> Result<NodeIdentity, String> {
-    let private_key = PrivateKey::random(&mut OsRng);
-    let node_identity = NodeIdentity::new(
-        private_key,
+    let node_identity = NodeIdentity::random(
+        &mut OsRng,
         match public_addr {
             Some(public_addr) => public_addr,
             None => format!("{}/tcp/18141", get_local_ip().ok_or("Can't get local ip address")?)
@@ -166,26 +170,6 @@ pub fn create_new_identity<P: AsRef<Path>>(
     Ok(node_identity)
 }
 
-/// Recover a node id from a given private key and save it to disk
-/// ## Parameters
-/// `private_key` - The private key
-/// `path` - Reference to path to save the file
-/// `public_addr` - Network address of the base node
-/// `peer_features` - The features enabled for the base node
-///
-/// ## Returns
-/// A NodeIdentity wrapped in an atomic reference counter on success, the exit code indicating the reason on failure
-pub fn recover_node_identity<P: AsRef<Path>>(
-    private_key: PrivateKey,
-    path: P,
-    public_addr: &Multiaddr,
-    features: PeerFeatures,
-) -> Result<Arc<NodeIdentity>, ExitCodes> {
-    let node_identity = NodeIdentity::new(private_key, public_addr.clone(), features);
-    save_as_json(path, &node_identity).map_err(ExitCodes::IOError)?;
-    Ok(Arc::new(node_identity))
-}
-
 /// Loads the node identity from json at the given path
 /// ## Parameters
 /// `path` - Path to file from which to load the node identity
@@ -213,15 +197,14 @@ pub fn load_from_json<P: AsRef<Path>, T: DeserializeOwned>(path: P) -> Result<T,
 /// ## Returns
 /// Result to check if successful or not, string will indicate reason on error
 pub fn save_as_json<P: AsRef<Path>, T: Serialize>(path: P, object: &T) -> Result<(), String> {
-    let json = json5::to_string(object).unwrap();
+    let json = json5::to_string(object).map_err(|err| err.to_string())?;
     if let Some(p) = path.as_ref().parent() {
         if !p.exists() {
             fs::create_dir_all(p).map_err(|e| format!("Could not save json to data folder. {}", e))?;
         }
     }
     let json_with_comment = format!(
-        "// The public address is overwritten by the config/environment variable \
-         (TARI_BASE_NODE__<network>__PUBLIC_ADDRESS)\n{}",
+        "// This file is generated by the Tari base node. Any changes will be overwritten.\n{}",
         json
     );
     fs::write(path.as_ref(), json_with_comment.as_bytes()).map_err(|e| {

applications/tari_base_node/src/command_handler.rs

@@ -419,7 +419,7 @@ impl CommandHandler {
             let peer = match peer_manager.find_all_starts_with(&partial).await {
                 Ok(peers) if peers.is_empty() => {
                     if let Some(pk) = parse_emoji_id_or_public_key(&original_str) {
-                        if let Ok(peer) = peer_manager.find_by_public_key(&pk).await {
+                        if let Ok(Some(peer)) = peer_manager.find_by_public_key(&pk).await {
                             peer
                         } else {
                             println!("No peer matching '{}'", original_str);
@@ -457,6 +457,9 @@ impl CommandHandler {
             if let Some(dt) = peer.last_seen() {
                 println!("Last seen: {}", dt);
             }
+            if let Some(updated_at) = peer.identity_signature.map(|i| i.updated_at()) {
+                println!("Last updated: {} (UTC)", updated_at);
+            }
         });
     }
 
@@ -479,7 +482,7 @@ impl CommandHandler {
                     let num_peers = peers.len();
                     println!();
                     let mut table = Table::new();
-                    table.set_titles(vec!["NodeId", "Public Key", "Flags", "Role", "User Agent", "Info"]);
+                    table.set_titles(vec!["NodeId", "Public Key", "Role", "User Agent", "Info"]);
 
                     for peer in peers {
                         let info_str = {
@@ -491,7 +494,7 @@ impl CommandHandler {
                                 }
                             } else if let Some(dt) = peer.last_seen() {
                                 s.push(format!(
-                                    "LAST_SEEN = {}",
+                                    "LAST_SEEN: {}",
                                     Utc::now()
                                         .naive_utc()
                                         .signed_duration_since(dt)
@@ -516,10 +519,11 @@ impl CommandHandler {
                                 .get_metadata(1)
                                 .and_then(|v| bincode::deserialize::<PeerMetadata>(v).ok())
                             {
-                                s.push(format!(
-                                    "chain height = {}",
-                                    metadata.metadata.height_of_longest_chain()
-                                ));
+                                s.push(format!("chain height: {}", metadata.metadata.height_of_longest_chain()));
+                            }
+
+                            if let Some(updated_at) = peer.identity_signature.map(|i| i.updated_at()) {
+                                s.push(format!("updated_at: {} (UTC)", updated_at));
                             }
 
                             if s.is_empty() {
@@ -531,7 +535,6 @@ impl CommandHandler {
                         table.add_row(row![
                             peer.node_id,
                             peer.public_key,
-                            format!("{:?}", peer.flags),
                             {
                                 if peer.features == PeerFeatures::COMMUNICATION_CLIENT {
                                     "Wallet"
@@ -724,7 +727,8 @@ impl CommandHandler {
                         let peer = peer_manager
                             .find_by_node_id(conn.peer_node_id())
                             .await
-                            .expect("Unexpected peer database error or peer not found");
+                            .expect("Unexpected peer database error")
+                            .expect("Peer not found");
 
                         let chain_height = peer
                             .get_metadata(1)

applications/tari_base_node/src/grpc/base_node_grpc_server.rs

@@ -1415,7 +1415,8 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
                 peer_manager
                     .find_by_node_id(peer.peer_node_id())
                     .await
-                    .map_err(|err| Status::internal(err.to_string()))?,
+                    .map_err(|err| Status::internal(err.to_string()))?
+                    .ok_or_else(|| Status::not_found(format!("Peer {} not found", peer.peer_node_id())))?,
             );
         }
 

applications/tari_console_wallet/src/automation/commands.rs

@@ -310,7 +310,7 @@ async fn set_base_node_peer(
     println!("Setting base node peer...");
     println!("{}::{}", public_key, net_address);
     wallet
-        .set_base_node_peer(public_key.clone(), net_address.to_string())
+        .set_base_node_peer(public_key.clone(), net_address.clone())
         .await?;
     Ok((public_key, net_address))
 }

applications/tari_console_wallet/src/grpc/wallet_grpc_server.rs

@@ -819,12 +819,13 @@ impl wallet_server::Wallet for WalletGrpcServer {
             .map_err(|err| Status::internal(err.to_string()))?;
 
         let mut peers = Vec::with_capacity(connected_peers.len());
-        for peer in connected_peers {
+        for conn in connected_peers {
             peers.push(
                 peer_manager
-                    .find_by_node_id(peer.peer_node_id())
+                    .find_by_node_id(conn.peer_node_id())
                     .await
-                    .map_err(|err| Status::internal(err.to_string()))?,
+                    .map_err(|err| Status::internal(err.to_string()))?
+                    .ok_or_else(|| Status::not_found(format!("Peer '{}' not found", conn.peer_node_id())))?,
             );
         }
 

applications/tari_console_wallet/src/init/mod.rs

@@ -30,11 +30,12 @@ use tari_common::{exit_codes::ExitCodes, ConfigBootstrap, GlobalConfig};
 use tari_comms::{
     multiaddr::Multiaddr,
     peer_manager::{Peer, PeerFeatures},
-    types::CommsSecretKey,
+    types::CommsPublicKey,
     NodeIdentity,
 };
 use tari_comms_dht::{store_forward::SafConfig, DbConnectionUrl, DhtConfig};
 use tari_core::transactions::CryptoFactories;
+use tari_crypto::keys::PublicKey;
 use tari_key_manager::cipher_seed::CipherSeed;
 use tari_p2p::{
     auto_update::AutoUpdateConfig,
@@ -50,6 +51,7 @@ use tari_wallet::{
     output_manager_service::config::OutputManagerServiceConfig,
     storage::{database::WalletDatabase, sqlite_utilities::initialize_sqlite_database_backends},
     transaction_service::config::{TransactionRoutingMechanism, TransactionServiceConfig},
+    wallet::{derive_comms_secret_key, read_or_create_master_seed},
     Wallet,
     WalletConfig,
     WalletSqlite,
@@ -309,24 +311,51 @@ pub async fn init_wallet(
         "Databases Initialized. Wallet encrypted? {}.", wallet_encrypted
     );
 
-    let node_address = match wallet_db.get_node_address().await? {
-        None => match config.public_address.clone() {
-            Some(val) => val,
+    let node_address = match config.public_address.clone() {
+        Some(addr) => addr,
+        None => match wallet_db.get_node_address().await? {
+            Some(addr) => addr,
             None => Multiaddr::empty(),
         },
-        Some(a) => a,
     };
 
-    let node_features = match wallet_db.get_node_features().await? {
-        None => PeerFeatures::COMMUNICATION_CLIENT,
-        Some(nf) => nf,
-    };
+    let node_features = wallet_db
+        .get_node_features()
+        .await?
+        .unwrap_or(PeerFeatures::COMMUNICATION_CLIENT);
+
+    let identity_sig = wallet_db.get_comms_identity_signature().await?;
+
+    let master_seed = read_or_create_master_seed(recovery_seed.clone(), &wallet_db).await?;
+    let comms_secret_key = derive_comms_secret_key(&master_seed)?;
 
-    let node_identity = Arc::new(NodeIdentity::new(
-        CommsSecretKey::default(),
+    // This checks if anything has changed by validating the previous signature and if invalid, setting identity_sig to
+    // None
+    let identity_sig = identity_sig.filter(|sig| {
+        let comms_public_key = CommsPublicKey::from_secret_key(&comms_secret_key);
+        sig.is_valid(&comms_public_key, node_features, [&node_address])
+    });
+
+    // SAFETY: we are manually checking the validity of this signature before adding Some(..)
+    let node_identity = Arc::new(NodeIdentity::with_signature_unchecked(
+        comms_secret_key,
         node_address,
         node_features,
+        identity_sig,
     ));
+    if !node_identity.is_signed() {
+        node_identity.sign();
+        // unreachable panic: signed above
+        wallet_db
+            .set_comms_identity_signature(
+                node_identity
+                    .identity_signature_read()
+                    .as_ref()
+                    .expect("unreachable panic")
+                    .clone(),
+            )
+            .await?;
+    }
 
     let transport_type = create_transport_type(config);
     let transport_type = match transport_type {
@@ -425,7 +454,7 @@ pub async fn init_wallet(
         output_manager_backend,
         contacts_backend,
         shutdown_signal,
-        recovery_seed.clone(),
+        master_seed,
     )
     .await
     .map_err(|e| {
@@ -499,11 +528,10 @@ pub async fn start_wallet(
     let net_address = base_node
         .addresses
         .first()
-        .ok_or_else(|| ExitCodes::ConfigError("Configured base node has no address!".to_string()))?
-        .to_string();
+        .ok_or_else(|| ExitCodes::ConfigError("Configured base node has no address!".to_string()))?;
 
     wallet
-        .set_base_node_peer(base_node.public_key.clone(), net_address)
+        .set_base_node_peer(base_node.public_key.clone(), net_address.address.clone())
         .await
         .map_err(|e| ExitCodes::WalletError(format!("Error setting wallet base node peer. {}", e)))?;