From 4c6e6c01a2726fb37f35cf0bdb180a84a527e2f4 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:24:06 +0530 Subject: [PATCH 01/52] Add `libp2p` as a dependency. --- Cargo.lock | 3 +++ crates/spartan-farmer/Cargo.toml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ac7a2977dd..d94cc6997d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3213,12 +3213,14 @@ dependencies = [ "async-io", "futures 0.3.17", "futures-timer 3.0.2", + "if-addrs", "if-watch", "ipnet", "libc", "libp2p-core", "log", "socket2 0.4.1", + "tokio", ] [[package]] @@ -7358,6 +7360,7 @@ dependencies = [ "hex", "indicatif", "jsonrpsee", + "libp2p", "log", "rand 0.8.4", "rayon", diff --git a/crates/spartan-farmer/Cargo.toml b/crates/spartan-farmer/Cargo.toml index c667bea99c..0c803bc28f 100644 --- a/crates/spartan-farmer/Cargo.toml +++ b/crates/spartan-farmer/Cargo.toml @@ -48,6 +48,11 @@ version = "0.16.0" features = ["derive"] version = "1.0.125" +[dependencies.libp2p] +version = "0.39.1" +features = [ "noise", "mplex", "kad", "tcp-tokio" ] +default-features = false + [dev-dependencies] rand = "0.8.3" From cd7f3d1a4ebf18af2d0770dba25a2b43f5603d82 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:29:08 +0530 Subject: [PATCH 02/52] Add `dht` module and sub-modules for `client`, `eventloop` and `core`. --- crates/spartan-farmer/src/commands.rs | 1 + crates/spartan-farmer/src/commands/dht.rs | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 crates/spartan-farmer/src/commands/dht.rs diff --git a/crates/spartan-farmer/src/commands.rs b/crates/spartan-farmer/src/commands.rs index d700585e86..b316d4114a 100644 --- a/crates/spartan-farmer/src/commands.rs +++ b/crates/spartan-farmer/src/commands.rs @@ -1,3 +1,4 @@ +pub mod dht; mod farm; mod plot; diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs new file mode 100644 index 0000000000..469c9f1747 --- /dev/null +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -0,0 +1,8 @@ +// The Client API which the end-user is supposed to interact with. +pub mod client; +// Core libp2p activities like defining network behaviour and events, bootstrap-ing, +// creating of swarm and such... +mod core; +// EventLoop which actually processes libp2p SwarmEvents. The Client API interacts with the +// EventLoop to transfer and receieve data. +mod eventloop; From a223c530e43f8b9b1aa9d271da193d50c4bc97ba Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:32:07 +0530 Subject: [PATCH 03/52] Add `libp2p` imports for creating ComposedBehaviour in `core` --- crates/spartan-farmer/src/commands/dht.rs | 18 ++++++++++++++++++ crates/spartan-farmer/src/commands/dht/core.rs | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 crates/spartan-farmer/src/commands/dht/core.rs diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 469c9f1747..337e6b4df3 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -1,3 +1,21 @@ +// Stuff for Kademlia +use libp2p::kad::{record::store::MemoryStore, Kademlia, KademliaEvent}; + +// Stuff for defining composed behaviour +use libp2p::NetworkBehaviour; + +// Stuff needed to create the swarm +use libp2p::core::{upgrade, Transport}; +use libp2p::identity; +use libp2p::mplex; +use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; +use libp2p::swarm::SwarmBuilder; +use libp2p::tcp::TokioTcpConfig; +use libp2p::{PeerId, Swarm}; + +// Stuff needed to set up channels between Client API task and EventLoop task. +use tokio::sync::mpsc::{channel, Receiver, Sender}; + // The Client API which the end-user is supposed to interact with. pub mod client; // Core libp2p activities like defining network behaviour and events, bootstrap-ing, diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs new file mode 100644 index 0000000000..fbc13c24b3 --- /dev/null +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -0,0 +1,18 @@ +// Pull imports from the parent module +use super::*; + +#[derive(NetworkBehaviour)] +#[behaviour(event_process = false, out_event = "ComposedEvent")] +pub struct ComposedBehaviour { + kademlia: Kademlia, +} + +pub enum ComposedEvent { + Kademlia(KademliaEvent), +} + +impl From for ComposedEvent { + fn from(event: KademliaEvent) -> Self { + ComposedEvent::Kademlia(event) + } +} From 06b596a453e7e23c5bcd5afeac10773909aeb9ae Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:33:24 +0530 Subject: [PATCH 04/52] Add method to create swarm for farmer. --- .../spartan-farmer/src/commands/dht/core.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index fbc13c24b3..09f8f42829 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -16,3 +16,29 @@ impl From for ComposedEvent { ComposedEvent::Kademlia(event) } } + +pub async fn create_swarm() -> Swarm { + // Generate IDs. + let key = identity::Keypair::generate_ed25519(); + let peerid = PeerId::from_public_key(key.public()); + + // Generate NOISE authentication keys. + let auth_keys = Keypair::::new().into_authentic(&key).unwrap(); + + // Create secure-TCP transport that uses tokio under the hood. + let transport = TokioTcpConfig::new() + .upgrade(upgrade::Version::V1) + .authenticate(NoiseConfig::xx(auth_keys).into_authenticated()) + .multiplex(mplex::MplexConfig::new()) + .boxed(); + + let behaviour = ComposedBehaviour { + kademlia: Kademlia::new(peerid.clone(), MemoryStore::new(peerid.clone())), + }; + + SwarmBuilder::new(transport, behaviour, peerid.clone()) + .executor(Box::new(|fut| { + tokio::spawn(fut); + })) + .build() +} From d73b610fa959b627da699cfe705965eb4127c9c8 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:36:00 +0530 Subject: [PATCH 05/52] Add code for `client` module. --- .../spartan-farmer/src/commands/dht/client.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 crates/spartan-farmer/src/commands/dht/client.rs diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs new file mode 100644 index 0000000000..d6f7d12181 --- /dev/null +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -0,0 +1,31 @@ +use super::core::{create_swarm, ComposedEvent}; +use super::eventloop::EventLoop; +use super::*; + +pub enum ClientEvent { + StartListening, + Dial, + Provide, + Find, +} + +pub struct Client { + network_rx: Receiver, + client_tx: Sender, +} + +impl Client { + /// This method will construct a new Swarm and eventloop. + pub async fn new() -> (Self, EventLoop) { + let (network_tx, network_rx) = channel(10); + let (client_tx, client_rx) = channel(10); + + return ( + Client { + network_rx, + client_tx, + }, + EventLoop::new(create_swarm().await, client_rx, network_tx) + ); + } +} From 45cc2cc25d22dece29ba082a8df8e1d9ae2ef861 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:37:43 +0530 Subject: [PATCH 06/52] Add code for `eventloop` module. --- .../src/commands/dht/eventloop.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 crates/spartan-farmer/src/commands/dht/eventloop.rs diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs new file mode 100644 index 0000000000..e8306660f7 --- /dev/null +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -0,0 +1,26 @@ +use super::client::ClientEvent; +use super::core::{ComposedBehaviour, ComposedEvent}; +use super::*; + +pub struct EventLoop { + swarm: Swarm, + client_rx: Receiver, + network_tx: Sender, +} + +impl EventLoop { + /// Create new event loop + pub fn new( + swarm: Swarm, + client_rx: Receiver, + network_tx: Sender, + ) -> Self { + EventLoop { + swarm, + client_rx, + network_tx, + } + } + /// Run event loop. We will use this method to spawn the event loop in a background task. + pub async fn run(&mut self) {} +} From 29c41b7328517856ca6fbbd5e0c647e1699ad1ab Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:55:41 +0530 Subject: [PATCH 07/52] Create a seperate method to create `Client` API and `EventLoop` objects --- .../spartan-farmer/src/commands/dht/client.rs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index d6f7d12181..3ed8df64e2 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -14,18 +14,24 @@ pub struct Client { client_tx: Sender, } +pub async fn dht_listener() -> (Client, EventLoop) { + let (network_tx, network_rx) = channel(10); + let (client_tx, client_rx) = channel(10); + + return ( + Client::new(network_rx, client_tx), + EventLoop::new(create_swarm().await, client_rx, network_tx), + ); +} + impl Client { /// This method will construct a new Swarm and eventloop. - pub async fn new() -> (Self, EventLoop) { - let (network_tx, network_rx) = channel(10); - let (client_tx, client_rx) = channel(10); - - return ( - Client { - network_rx, - client_tx, - }, - EventLoop::new(create_swarm().await, client_rx, network_tx) - ); + pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { + Client { + network_rx, + client_tx, + } + } + pub fn start_listening(&mut self) { } } From 9949e4f40dbefafb31e56b345fd82b9671c53214 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 00:56:51 +0530 Subject: [PATCH 08/52] Call DHT methods in `farm.rs` --- crates/spartan-farmer/src/commands/farm.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 92598ad0b9..b614de807c 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -14,6 +14,8 @@ use std::fs; use std::path::PathBuf; use std::time::Instant; +use super::dht::client as dht; + type SlotNumber = u64; #[derive(Debug, Serialize)] @@ -62,6 +64,20 @@ pub(crate) async fn farm(path: PathBuf, ws_server: &str) -> Result<(), Box Date: Sun, 19 Sep 2021 01:27:45 +0530 Subject: [PATCH 09/52] cargo format --- crates/spartan-farmer/src/commands/dht.rs | 2 +- crates/spartan-farmer/src/commands/dht/client.rs | 7 +++---- crates/spartan-farmer/src/commands/farm.rs | 4 +--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 337e6b4df3..3681d810a4 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -11,7 +11,7 @@ use libp2p::mplex; use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; use libp2p::swarm::SwarmBuilder; use libp2p::tcp::TokioTcpConfig; -use libp2p::{PeerId, Swarm}; +use libp2p::{Multiaddr, PeerId, Swarm}; // Stuff needed to set up channels between Client API task and EventLoop task. use tokio::sync::mpsc::{channel, Receiver, Sender}; diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 3ed8df64e2..76207369dc 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -3,7 +3,7 @@ use super::eventloop::EventLoop; use super::*; pub enum ClientEvent { - StartListening, + Listen, Dial, Provide, Find, @@ -14,6 +14,7 @@ pub struct Client { client_tx: Sender, } +/// This method will construct a new Swarm and EventLoop object. pub async fn dht_listener() -> (Client, EventLoop) { let (network_tx, network_rx) = channel(10); let (client_tx, client_rx) = channel(10); @@ -25,13 +26,11 @@ pub async fn dht_listener() -> (Client, EventLoop) { } impl Client { - /// This method will construct a new Swarm and eventloop. pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { Client { network_rx, client_tx, } } - pub fn start_listening(&mut self) { - } + pub fn start_listening(&mut self) {} } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index b614de807c..b198c11004 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -71,9 +71,7 @@ pub(crate) async fn farm(path: PathBuf, ws_server: &str) -> Result<(), Box Date: Sun, 19 Sep 2021 02:24:26 +0530 Subject: [PATCH 10/52] Add `select!` for running eventloop for client and swarm events. --- crates/spartan-farmer/src/commands/dht.rs | 4 +++- crates/spartan-farmer/src/commands/dht/eventloop.rs | 9 ++++++++- crates/spartan-farmer/src/commands/farm.rs | 9 ++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 3681d810a4..79804219af 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -14,7 +14,9 @@ use libp2p::tcp::TokioTcpConfig; use libp2p::{Multiaddr, PeerId, Swarm}; // Stuff needed to set up channels between Client API task and EventLoop task. -use tokio::sync::mpsc::{channel, Receiver, Sender}; +use futures::channel::mpsc::{channel, Receiver, Sender}; +use futures::prelude::*; +use futures::StreamExt; // The Client API which the end-user is supposed to interact with. pub mod client; diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index e8306660f7..1ca3f55530 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -22,5 +22,12 @@ impl EventLoop { } } /// Run event loop. We will use this method to spawn the event loop in a background task. - pub async fn run(&mut self) {} + pub async fn run(mut self) { + loop { + futures::select! { + client = self.client_rx.next() => {}, + swarm = self.swarm.next() => {}, + } + } + } } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index b198c11004..4adbb930e7 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -66,15 +66,18 @@ pub(crate) async fn farm(path: PathBuf, ws_server: &str) -> Result<(), Box Date: Sun, 19 Sep 2021 02:25:49 +0530 Subject: [PATCH 11/52] Add `start_listening` method in `Client` API. --- crates/spartan-farmer/src/commands/dht/client.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 76207369dc..0874620a6b 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -2,8 +2,9 @@ use super::core::{create_swarm, ComposedEvent}; use super::eventloop::EventLoop; use super::*; +#[derive(Debug)] pub enum ClientEvent { - Listen, + Listen { addr: Multiaddr }, Dial, Provide, Find, @@ -32,5 +33,11 @@ impl Client { client_tx, } } - pub fn start_listening(&mut self) {} + + pub async fn start_listening(&mut self, addr: Multiaddr) { + self.client_tx + .send(ClientEvent::Listen { addr }) + .await + .expect("Listening failed."); + } } From d5a021eabfbdc013af4748f5d8aa3aa5f7754a0f Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 15:33:10 +0530 Subject: [PATCH 12/52] Add `ClientConfig` for bootstrapping --- .../spartan-farmer/src/commands/dht/client.rs | 29 ++++++++++++------- .../spartan-farmer/src/commands/dht/core.rs | 2 +- crates/spartan-farmer/src/commands/farm.rs | 12 ++++++-- crates/spartan-farmer/src/main.rs | 7 ++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 0874620a6b..72822d356a 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -6,6 +6,7 @@ use super::*; pub enum ClientEvent { Listen { addr: Multiaddr }, Dial, + Bootstrap, Provide, Find, } @@ -15,17 +16,6 @@ pub struct Client { client_tx: Sender, } -/// This method will construct a new Swarm and EventLoop object. -pub async fn dht_listener() -> (Client, EventLoop) { - let (network_tx, network_rx) = channel(10); - let (client_tx, client_rx) = channel(10); - - return ( - Client::new(network_rx, client_tx), - EventLoop::new(create_swarm().await, client_rx, network_tx), - ); -} - impl Client { pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { Client { @@ -41,3 +31,20 @@ impl Client { .expect("Listening failed."); } } + +pub struct ClientConfig { + // This will be true, if we are running a bootstrap node. + pub bootstrap: bool, +} + +/// This method will construct a new Swarm and EventLoop object. +pub async fn dht_listener(config: ClientConfig) -> (Client, EventLoop) { + let (network_tx, network_rx) = channel(10); + let (client_tx, client_rx) = channel(10); + + return ( + Client::new(network_rx, client_tx), + EventLoop::new(create_swarm(config.bootstrap).await, client_rx, network_tx), + ); +} + diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 09f8f42829..1e6ef0a125 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -17,7 +17,7 @@ impl From for ComposedEvent { } } -pub async fn create_swarm() -> Swarm { +pub async fn create_swarm(bootstrap: bool) -> Swarm { // Generate IDs. let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 4adbb930e7..e7593f9156 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -14,7 +14,7 @@ use std::fs; use std::path::PathBuf; use std::time::Instant; -use super::dht::client as dht; +use super::dht::{client as dht, client::ClientConfig}; type SlotNumber = u64; @@ -55,7 +55,11 @@ struct SlotInfo { /// Start farming by using plot in specified path and connecting to WebSocket server at specified /// address. -pub(crate) async fn farm(path: PathBuf, ws_server: &str) -> Result<(), Box> { +pub(crate) async fn farm( + bootstrap: bool, + path: PathBuf, + ws_server: &str, +) -> Result<(), Box> { info!("Connecting to RPC server"); let client = WsClientBuilder::default().build(ws_server).await?; @@ -69,7 +73,9 @@ pub(crate) async fn farm(path: PathBuf, ws_server: &str) -> Result<(), Box, #[clap(long, default_value = "ws://127.0.0.1:9944")] ws_server: String, + #[clap(short, long)] + bootstrap: bool, }, } @@ -99,10 +101,13 @@ fn main() { Command::Farm { custom_path, ws_server, + bootstrap, } => { let path = utils::get_path(custom_path); let runtime = Runtime::new().unwrap(); - runtime.block_on(commands::farm(path, &ws_server)).unwrap(); + runtime + .block_on(commands::farm(bootstrap, path, &ws_server)) + .unwrap(); } } } From 22299140fb47dfa0f941f4de332d31afa93dcc27 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 17:52:03 +0530 Subject: [PATCH 13/52] Implement `listen_on` in `Client` API and `EventLoop` --- crates/spartan-farmer/src/commands/dht.rs | 4 +- .../spartan-farmer/src/commands/dht/client.rs | 16 ++++++-- .../src/commands/dht/eventloop.rs | 41 ++++++++++++++++--- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 79804219af..90a34921a5 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -9,7 +9,7 @@ use libp2p::core::{upgrade, Transport}; use libp2p::identity; use libp2p::mplex; use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; -use libp2p::swarm::SwarmBuilder; +use libp2p::swarm::{SwarmBuilder, SwarmEvent}; use libp2p::tcp::TokioTcpConfig; use libp2p::{Multiaddr, PeerId, Swarm}; @@ -18,6 +18,8 @@ use futures::channel::mpsc::{channel, Receiver, Sender}; use futures::prelude::*; use futures::StreamExt; +use log::info; + // The Client API which the end-user is supposed to interact with. pub mod client; // Core libp2p activities like defining network behaviour and events, bootstrap-ing, diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 72822d356a..3380692069 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,10 +1,11 @@ -use super::core::{create_swarm, ComposedEvent}; +use super::core::{create_swarm, ComposedBehaviour, ComposedEvent}; use super::eventloop::EventLoop; use super::*; #[derive(Debug)] pub enum ClientEvent { Listen { addr: Multiaddr }, + ReturnListen { addr: Multiaddr }, Dial, Bootstrap, Provide, @@ -12,12 +13,12 @@ pub enum ClientEvent { } pub struct Client { - network_rx: Receiver, + network_rx: Receiver, client_tx: Sender, } impl Client { - pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { + pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { Client { network_rx, client_tx, @@ -48,3 +49,12 @@ pub async fn dht_listener(config: ClientConfig) -> (Client, EventLoop) { ); } +pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { + match event { + ClientEvent::Listen { addr } => match swarm.listen_on(addr) { + Ok(_) => {} + Err(e) => info!("{:?}", e), + }, + _ => {} + } +} diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 1ca3f55530..369dc14e10 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -1,11 +1,11 @@ -use super::client::ClientEvent; +use super::client::{handle_client_event, ClientEvent}; use super::core::{ComposedBehaviour, ComposedEvent}; use super::*; pub struct EventLoop { swarm: Swarm, client_rx: Receiver, - network_tx: Sender, + network_tx: Sender, } impl EventLoop { @@ -13,7 +13,7 @@ impl EventLoop { pub fn new( swarm: Swarm, client_rx: Receiver, - network_tx: Sender, + network_tx: Sender, ) -> Self { EventLoop { swarm, @@ -21,13 +21,44 @@ impl EventLoop { network_tx, } } + /// Run event loop. We will use this method to spawn the event loop in a background task. pub async fn run(mut self) { loop { futures::select! { - client = self.client_rx.next() => {}, - swarm = self.swarm.next() => {}, + client_event = self.client_rx.next() => self.handle_event(client_event.unwrap()), + network_event = self.swarm.next() => match network_event { + Some(event) => self.handle_network_event(event).await, + None => break, + } + } + } + } + + // NOTE: This method is just a wrapper around the actual method that will handle client events. + // We have to put the handle client event method in the EventLoop impl because it needs access + // to the swarm. + fn handle_event(&mut self, event: ClientEvent) { + handle_client_event(&mut self.swarm, event) + } + + async fn handle_network_event(&mut self, event: SwarmEvent) { + match event { + SwarmEvent::Behaviour(event) => match event { + ComposedEvent::Kademlia(event) => match event {}, + }, + SwarmEvent::UnreachableAddr { .. } => todo!(), + SwarmEvent::NewListenAddr { address, .. } => { + self.network_tx + .send(ClientEvent::ReturnListen { + addr: address.clone(), + }) + .await + .unwrap(); + info!("Farmer is listening to K-DHT on: {:?}", address) } + SwarmEvent::Dialing(_) => todo!(), + _ => {} } } } From 204675a3e60cce9a361bcb55bfbf130fec76b734 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Sun, 19 Sep 2021 18:05:50 +0530 Subject: [PATCH 14/52] fix clippy error --- crates/spartan-farmer/src/commands/dht/eventloop.rs | 6 ++++-- crates/spartan-farmer/src/commands/farm.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 369dc14e10..2914c7127f 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -45,9 +45,10 @@ impl EventLoop { async fn handle_network_event(&mut self, event: SwarmEvent) { match event { SwarmEvent::Behaviour(event) => match event { - ComposedEvent::Kademlia(event) => match event {}, + ComposedEvent::Kademlia(event) => match event { + _ => {} + }, }, - SwarmEvent::UnreachableAddr { .. } => todo!(), SwarmEvent::NewListenAddr { address, .. } => { self.network_tx .send(ClientEvent::ReturnListen { @@ -57,6 +58,7 @@ impl EventLoop { .unwrap(); info!("Farmer is listening to K-DHT on: {:?}", address) } + SwarmEvent::UnreachableAddr { .. } => todo!(), SwarmEvent::Dialing(_) => todo!(), _ => {} } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index e7593f9156..e2c74aa9b3 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -75,7 +75,7 @@ pub(crate) async fn farm( // 3. The task will run an eventloop and keep discovering new peers. let config = ClientConfig { bootstrap }; - let (mut dht_client, mut dht_eventloop) = dht::dht_listener(config).await; + let (mut dht_client, dht_eventloop) = dht::dht_listener(config).await; tokio::spawn(async move { dht_eventloop.run().await }); From c58af026e738ed51062a32b195bfe773356bb8a4 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Mon, 20 Sep 2021 01:01:13 +0530 Subject: [PATCH 15/52] remove rustfmt.toml --- crates/spartan-farmer/rustfmt.toml | 75 ------------------------------ 1 file changed, 75 deletions(-) delete mode 100644 crates/spartan-farmer/rustfmt.toml diff --git a/crates/spartan-farmer/rustfmt.toml b/crates/spartan-farmer/rustfmt.toml deleted file mode 100644 index 048d644b35..0000000000 --- a/crates/spartan-farmer/rustfmt.toml +++ /dev/null @@ -1,75 +0,0 @@ -max_width = 100 -hard_tabs = false -tab_spaces = 4 -newline_style = "Auto" -indent_style = "Block" -use_small_heuristics = "Default" -fn_call_width = 60 -attr_fn_like_width = 70 -struct_lit_width = 18 -struct_variant_width = 35 -array_width = 60 -chain_width = 60 -single_line_if_else_max_width = 50 -wrap_comments = false -format_code_in_doc_comments = false -comment_width = 80 -normalize_comments = false -normalize_doc_attributes = false -license_template_path = "" -format_strings = false -format_macro_matchers = false -format_macro_bodies = true -empty_item_single_line = true -struct_lit_single_line = true -fn_single_line = false -where_single_line = false -imports_indent = "Block" -imports_layout = "Mixed" -imports_granularity = "Preserve" -group_imports = "Preserve" -reorder_imports = true -reorder_modules = true -reorder_impl_items = false -type_punctuation_density = "Wide" -space_before_colon = false -space_after_colon = true -spaces_around_ranges = false -binop_separator = "Front" -remove_nested_parens = true -combine_control_expr = true -overflow_delimited_expr = false -struct_field_align_threshold = 0 -enum_discrim_align_threshold = 0 -match_arm_blocks = true -match_arm_leading_pipes = "Never" -force_multiline_blocks = false -fn_args_layout = "Tall" -brace_style = "SameLineWhere" -control_brace_style = "AlwaysSameLine" -trailing_semicolon = true -trailing_comma = "Vertical" -match_block_trailing_comma = false -blank_lines_upper_bound = 1 -blank_lines_lower_bound = 0 -edition = "2018" -version = "One" -inline_attribute_width = 0 -merge_derives = true -use_try_shorthand = false -use_field_init_shorthand = false -force_explicit_abi = true -condense_wildcard_suffixes = false -color = "Auto" -required_version = "1.4.37" -unstable_features = false -disable_all_formatting = false -skip_children = false -hide_parse_errors = false -error_on_line_overflow = false -error_on_unformatted = false -report_todo = "Never" -report_fixme = "Never" -ignore = [] -emit_mode = "Files" -make_backup = false From 0f0c738521602ecf58e3232276881f7de506a10d Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Mon, 20 Sep 2021 12:45:44 +0530 Subject: [PATCH 16/52] Bootstrap implementation, part 2. 1. Update `oneshot` to receive errors. 2. Add `ClientType` enum. 3. Add methods to create bootstrap node. --- crates/spartan-farmer/src/commands/dht.rs | 8 +- .../spartan-farmer/src/commands/dht/client.rs | 75 +++++++++++++------ .../spartan-farmer/src/commands/dht/core.rs | 47 ++++-------- .../src/commands/dht/eventloop.rs | 27 ++----- crates/spartan-farmer/src/commands/farm.rs | 31 ++++---- 5 files changed, 93 insertions(+), 95 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 90a34921a5..9c7b00a7bf 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -14,12 +14,18 @@ use libp2p::tcp::TokioTcpConfig; use libp2p::{Multiaddr, PeerId, Swarm}; // Stuff needed to set up channels between Client API task and EventLoop task. -use futures::channel::mpsc::{channel, Receiver, Sender}; +use futures::channel::{ + mpsc::{channel, Receiver, Sender}, + oneshot, +}; use futures::prelude::*; use futures::StreamExt; use log::info; +type OneshotError = Box; +type OneshotType = Result<(), OneshotError>; + // The Client API which the end-user is supposed to interact with. pub mod client; // Core libp2p activities like defining network behaviour and events, bootstrap-ing, diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 97528374e6..2b1583393e 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,58 +1,89 @@ -use super::core::{create_swarm, ComposedBehaviour}; +use super::core::{create_bootstrap, create_node, ComposedBehaviour}; use super::eventloop::EventLoop; use super::*; #[derive(Debug)] pub enum ClientEvent { - Listen { addr: Multiaddr }, - ReturnListen { addr: Multiaddr }, - Dial, - Provide, - Find, + Listen { + addr: Multiaddr, + sender: oneshot::Sender, + }, } +pub enum ClientType { + Bootstrap, + Normal, +} + +// TODO: It would make sense to add an enum for the types of `Clients`: `Normal` and `Bootnode pub struct Client { - network_rx: Receiver, client_tx: Sender, } impl Client { - pub fn new(network_rx: Receiver, client_tx: Sender) -> Self { - Client { - network_rx, - client_tx, - } + pub fn new(client_tx: Sender) -> Self { + Client { client_tx } } pub async fn start_listening(&mut self, addr: Multiaddr) { + let (tx, rx) = oneshot::channel(); + self.client_tx - .send(ClientEvent::Listen { addr }) + .send(ClientEvent::Listen { addr, sender: tx }) .await - .expect("Listening failed."); + .unwrap(); + + rx.await.expect("Failed to start listening."); } + + pub async fn dial() {} } +pub type BootAddr = String; + pub struct ClientConfig { // This will be true, if we are running a bootstrap node. pub bootstrap: bool, + pub bootstrap_nodes: Vec, + // pub bootstrap_keys: Vec, + pub client_type: ClientType, } +// TODO: This method needs a new (and, better) name. /// This method will construct a new Swarm and EventLoop object. -pub async fn dht_listener(config: ClientConfig) -> (Client, EventLoop) { - let (network_tx, network_rx) = channel(10); +pub async fn create_connection(config: ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); - let client = Client::new(network_rx, client_tx); - let eventloop = EventLoop::new(create_swarm(config.bootstrap).await, client_rx, network_tx); + let client = Client::new(client_tx); + + // TODO: We need to split the creation of bootstrap nodes at this point, don't call create_swarm at + // all. + // - If, we have an enum for Client type, this can be made even better. + // - The seperate method to create bootnodes can be in the `Client` module and called by `farm.rs`. + // - We can read the ClientConfig file, in that method itself. + // - We can even it spawn another task for it. - return (client, eventloop); + match config.client_type { + ClientType::Bootstrap => { + let eventloop = EventLoop::new(create_bootstrap(config).await, client_rx); + return (client, eventloop); + } + ClientType::Normal => { + let eventloop = EventLoop::new(create_node(config).await, client_rx); + return (client, eventloop); + } + } } pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { match event { - ClientEvent::Listen { addr } => match swarm.listen_on(addr) { - Ok(_) => {} - Err(e) => info!("{:?}", e), + ClientEvent::Listen { addr, sender } => match swarm.listen_on(addr) { + Ok(_) => { + sender.send(Ok(())).unwrap(); + } + Err(e) => { + sender.send(Err(Box::new(e))).unwrap(); + } }, _ => {} } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 1b49e9368f..63692c8fc8 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -1,12 +1,7 @@ // Pull imports from the parent module +use super::client::ClientConfig; use super::*; -const BOOTNODES: [&str; 3] = [ - "/ip4/192.186.0.0/tcp/9997", - "/ip4/192.186.0.0/tcp/9998", - "/ip4/192.186.0.0/tcp/9999", -]; - #[derive(NetworkBehaviour)] #[behaviour(event_process = false, out_event = "ComposedEvent")] pub struct ComposedBehaviour { @@ -23,7 +18,16 @@ impl From for ComposedEvent { } } -pub async fn create_swarm(bootstrap: bool) -> Swarm { +pub async fn create_bootstrap(config: ClientConfig) -> Swarm { + // TODO: Don't do this, here. Move this to a seperate method. That can be called by + // dht::client::dht_listener. + // + // Read a RSA private key from disk, to create a bootstrap node's PeerID. + // let key = identity::Keypair::rsa_from_pkcs8().unwrap(); + todo!() +} + +pub async fn create_node(config: ClientConfig) -> Swarm { // Generate IDs. let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); @@ -48,34 +52,11 @@ pub async fn create_swarm(bootstrap: bool) -> Swarm { })) .build(); - if bootstrap { - set_bootstrap(&mut swarm); - } else { - dial_bootstrap(&mut swarm); - } + dial_bootstrap(&mut swarm, config.bootstrap_nodes); swarm } -fn set_bootstrap(swarm: &mut Swarm) { - info!("I'm a bootstrap node.\n"); - for node in &BOOTNODES { - let addr: Multiaddr = node.clone().parse().unwrap(); - match swarm.dial_addr(addr.clone()) { - Err(_) => { - swarm.listen_on(addr.clone()).unwrap(); - info!("My Peer ID is: {:?}\n", swarm.local_peer_id()); - info!("My address is: {:?}\n", addr); - break; - } - _ => continue, - } - } -} - -fn dial_bootstrap(swarm: &mut Swarm) { - for node in BOOTNODES { - let addr: Multiaddr = node.clone().parse().unwrap(); - // swarm.behaviour_mut().kademlia.add_address(addr); - } +fn dial_bootstrap(_swarm: &mut Swarm, nodes: Vec) { + todo!() } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 8e01c4c6cd..e9f6bbdd99 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -5,21 +5,12 @@ use super::*; pub struct EventLoop { swarm: Swarm, client_rx: Receiver, - network_tx: Sender, } impl EventLoop { /// Create new event loop - pub fn new( - swarm: Swarm, - client_rx: Receiver, - network_tx: Sender, - ) -> Self { - EventLoop { - swarm, - client_rx, - network_tx, - } + pub fn new(swarm: Swarm, client_rx: Receiver) -> Self { + EventLoop { swarm, client_rx } } /// Run event loop. We will use this method to spawn the event loop in a background task. @@ -35,9 +26,8 @@ impl EventLoop { } } - // NOTE: This method is just a wrapper around the actual method that will handle client events. - // We have to put the handle client event method in the EventLoop impl because it needs access - // to the swarm. + // NOTE: We have to put the handle client event method in the EventLoop impl because it + // needs access to the swarm. fn handle_event(&mut self, event: ClientEvent) { handle_client_event(&mut self.swarm, event) } @@ -50,15 +40,8 @@ impl EventLoop { }, }, SwarmEvent::NewListenAddr { address, .. } => { - self.network_tx - .send(ClientEvent::ReturnListen { - addr: address.clone(), - }) - .await - .unwrap(); + info!("Farmer is listening to K-DHT on: {:?}", address) } - SwarmEvent::UnreachableAddr { .. } => todo!(), - SwarmEvent::Dialing(_) => todo!(), _ => {} } } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index d87980f817..c1ac8d71ac 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -14,7 +14,10 @@ use std::fs; use std::path::PathBuf; use std::time::Instant; -use super::dht::{client as dht, client::ClientConfig}; +use super::dht::{ + client as dht, + client::{ClientConfig, ClientType}, +}; type SlotNumber = u64; @@ -76,26 +79,20 @@ pub(crate) async fn farm( // 1. Create the swarm with the peer in it. // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. - let config = ClientConfig { bootstrap }; + let config = ClientConfig { + bootstrap, + bootstrap_nodes: Vec::default(), + client_type: ClientType::Normal, + }; - let (mut dht_client, dht_eventloop) = dht::dht_listener(config).await; + let (mut dht_client, dht_eventloop) = dht::create_connection(config).await; - if bootstrap { - dht_eventloop.run().await - } else { - tokio::spawn(async move { dht_eventloop.run().await }); - } + tokio::spawn(async move { dht_eventloop.run().await }); info!("Connecting to DHT"); - // If, I'm a bootstrap node, the lower-level `dht-core` code will take care of which address - // I'm supposed to listen on. - // If, I'm not a bootstrap node, I can listen on any random address/port. - if !bootstrap { - dht_client - .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) - .await; - } else { - } + dht_client + .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) + .await; info!("Opening existing keypair"); let keypair = From 2966a81cc9003a54efe3463852819ab4108f0464 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Mon, 20 Sep 2021 13:21:05 +0530 Subject: [PATCH 17/52] Bootstrap implementation, part 3. 1. Add `listen_addr` in `ClientConfig` for bootstrap nodes. 2. Add code in `dial_bootstrap` methods. 3. Add more comments. --- .../spartan-farmer/src/commands/dht/client.rs | 57 ++++++++----------- .../spartan-farmer/src/commands/dht/core.rs | 18 ++++-- .../src/commands/dht/eventloop.rs | 1 + crates/spartan-farmer/src/commands/farm.rs | 22 +++++-- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 2b1583393e..57a0867d29 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -4,6 +4,7 @@ use super::*; #[derive(Debug)] pub enum ClientEvent { + // Event for adding a listening address. Listen { addr: Multiaddr, sender: oneshot::Sender, @@ -11,12 +12,18 @@ pub enum ClientEvent { } pub enum ClientType { + // Bootstrap node. It uses the following fields from `ClientConfig`: + // 1. `bootstrap_keys`: Private keys/private key location to create bootstrap node peerId. + // 2. `listen_addr`: Listening address for Bootstrap node. Bootstrap, + // Normal node. It uses the following fields from `ClientConfig`: + // 1. `bootstrap_nodes`: Bootstrap nodes addresses that the normal node must connect to. + // For setting listening address, use client.start_listening. Normal, } -// TODO: It would make sense to add an enum for the types of `Clients`: `Normal` and `Bootnode pub struct Client { + // This channel sends events from Client to EventLoop. client_tx: Sender, } @@ -25,54 +32,39 @@ impl Client { Client { client_tx } } + // Set listening address for a particular Normal node. pub async fn start_listening(&mut self, addr: Multiaddr) { - let (tx, rx) = oneshot::channel(); + let (sender, recv) = oneshot::channel(); - self.client_tx - .send(ClientEvent::Listen { addr, sender: tx }) - .await - .unwrap(); + let listen = ClientEvent::Listen { addr, sender }; - rx.await.expect("Failed to start listening."); - } + self.client_tx.send(listen).await.unwrap(); - pub async fn dial() {} + // Check if the ListenEvent was processed, properly. + let _ = recv.await.expect("Failed to start listening."); + } } -pub type BootAddr = String; - pub struct ClientConfig { // This will be true, if we are running a bootstrap node. - pub bootstrap: bool, - pub bootstrap_nodes: Vec, - // pub bootstrap_keys: Vec, + pub bootstrap_nodes: Vec<(Multiaddr, PeerId)>, + pub bootstrap_keys: Vec>, pub client_type: ClientType, + pub listen_addr: Option, } -// TODO: This method needs a new (and, better) name. -/// This method will construct a new Swarm and EventLoop object. +// This method will construct a new Swarm and EventLoop object. pub async fn create_connection(config: ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); let client = Client::new(client_tx); - // TODO: We need to split the creation of bootstrap nodes at this point, don't call create_swarm at - // all. - // - If, we have an enum for Client type, this can be made even better. - // - The seperate method to create bootnodes can be in the `Client` module and called by `farm.rs`. - // - We can read the ClientConfig file, in that method itself. - // - We can even it spawn another task for it. + let eventloop = match config.client_type { + ClientType::Bootstrap => EventLoop::new(create_bootstrap(config).await, client_rx), + ClientType::Normal => EventLoop::new(create_node(config).await, client_rx), + }; - match config.client_type { - ClientType::Bootstrap => { - let eventloop = EventLoop::new(create_bootstrap(config).await, client_rx); - return (client, eventloop); - } - ClientType::Normal => { - let eventloop = EventLoop::new(create_node(config).await, client_rx); - return (client, eventloop); - } - } + return (client, eventloop); } pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { @@ -85,6 +77,5 @@ pub fn handle_client_event(swarm: &mut Swarm, event: ClientEv sender.send(Err(Box::new(e))).unwrap(); } }, - _ => {} } } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 63692c8fc8..2749f056fc 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -19,11 +19,16 @@ impl From for ComposedEvent { } pub async fn create_bootstrap(config: ClientConfig) -> Swarm { - // TODO: Don't do this, here. Move this to a seperate method. That can be called by - // dht::client::dht_listener. - // // Read a RSA private key from disk, to create a bootstrap node's PeerID. // let key = identity::Keypair::rsa_from_pkcs8().unwrap(); + // + // - We can read the ClientConfig file, in that method itself. + // - We can even it spawn another task for it. + + if let Some(_addr) = config.listen_addr { + // swarm.listen_on(addr).unwrap(); + } + todo!() } @@ -52,11 +57,14 @@ pub async fn create_node(config: ClientConfig) -> Swarm { })) .build(); + // Connect to bootstrap nodes. dial_bootstrap(&mut swarm, config.bootstrap_nodes); swarm } -fn dial_bootstrap(_swarm: &mut Swarm, nodes: Vec) { - todo!() +fn dial_bootstrap(swarm: &mut Swarm, nodes: Vec<(Multiaddr, PeerId)>) { + for node in nodes { + swarm.behaviour_mut().kademlia.add_address(&node.1, node.0); + } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index e9f6bbdd99..78fdf729f3 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -4,6 +4,7 @@ use super::*; pub struct EventLoop { swarm: Swarm, + // Channel to receive events from Client. client_rx: Receiver, } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index c1ac8d71ac..5037937cb5 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -79,20 +79,30 @@ pub(crate) async fn farm( // 1. Create the swarm with the peer in it. // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. + let node_type = if bootstrap { + ClientType::Bootstrap + } else { + ClientType::Normal + }; + let config = ClientConfig { - bootstrap, bootstrap_nodes: Vec::default(), - client_type: ClientType::Normal, + bootstrap_keys: Vec::default(), + client_type: node_type, + listen_addr: None, }; + info!("Connecting to DHT"); let (mut dht_client, dht_eventloop) = dht::create_connection(config).await; tokio::spawn(async move { dht_eventloop.run().await }); - info!("Connecting to DHT"); - dht_client - .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) - .await; + // For bootstrap nodes, we set the listening address through the `ClientConfig`. + if !bootstrap { + dht_client + .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) + .await; + } info!("Opening existing keypair"); let keypair = From 5977e6181bf3c0bb287a738b5e46fdb5d1ebac56 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Mon, 20 Sep 2021 20:35:08 +0530 Subject: [PATCH 18/52] Bootstrap implementation, Part 4. 1. Return Peer ID with `Client` object. 2. Add `enum` for `ClientType`. 3. Add methods to create bootstrap and normal nodes. --- .../spartan-farmer/src/commands/dht/client.rs | 77 ++++++++++--------- .../spartan-farmer/src/commands/dht/core.rs | 44 ++++++----- .../src/commands/dht/eventloop.rs | 4 +- crates/spartan-farmer/src/commands/farm.rs | 12 +-- 4 files changed, 73 insertions(+), 64 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 57a0867d29..857421e7cd 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -3,13 +3,6 @@ use super::eventloop::EventLoop; use super::*; #[derive(Debug)] -pub enum ClientEvent { - // Event for adding a listening address. - Listen { - addr: Multiaddr, - sender: oneshot::Sender, - }, -} pub enum ClientType { // Bootstrap node. It uses the following fields from `ClientConfig`: @@ -22,60 +15,70 @@ pub enum ClientType { Normal, } +pub enum ClientEvent { + // Event for adding a listening address. + Listen { + addr: Multiaddr, + sender: oneshot::Sender, + }, +} + +pub struct ClientConfig { + pub bootstrap_nodes: Vec<(Multiaddr, PeerId)>, + pub bootstrap_keys: Vec>, + pub client_type: ClientType, + pub listen_addr: Option, +} + pub struct Client { + pub peerid: PeerId, // This channel sends events from Client to EventLoop. client_tx: Sender, } impl Client { - pub fn new(client_tx: Sender) -> Self { - Client { client_tx } + pub fn new(peerid: PeerId, client_tx: Sender) -> Self { + Client { peerid, client_tx } } // Set listening address for a particular Normal node. pub async fn start_listening(&mut self, addr: Multiaddr) { let (sender, recv) = oneshot::channel(); - let listen = ClientEvent::Listen { addr, sender }; - - self.client_tx.send(listen).await.unwrap(); + self.client_tx + .send(ClientEvent::Listen { addr, sender }) + .await + .unwrap(); // Check if the ListenEvent was processed, properly. let _ = recv.await.expect("Failed to start listening."); } -} -pub struct ClientConfig { - // This will be true, if we are running a bootstrap node. - pub bootstrap_nodes: Vec<(Multiaddr, PeerId)>, - pub bootstrap_keys: Vec>, - pub client_type: ClientType, - pub listen_addr: Option, + pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { + match event { + ClientEvent::Listen { addr, sender } => match swarm.listen_on(addr) { + Ok(_) => { + sender.send(Ok(())).unwrap(); + } + Err(e) => { + sender.send(Err(Box::new(e))).unwrap(); + } + }, + } + } } // This method will construct a new Swarm and EventLoop object. pub async fn create_connection(config: ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); - let client = Client::new(client_tx); - - let eventloop = match config.client_type { - ClientType::Bootstrap => EventLoop::new(create_bootstrap(config).await, client_rx), - ClientType::Normal => EventLoop::new(create_node(config).await, client_rx), + let (peerid, swarm) = match config.client_type { + ClientType::Bootstrap => create_bootstrap(config).await, + ClientType::Normal => create_node(config).await, }; - return (client, eventloop); -} + let eventloop = EventLoop::new(swarm, client_rx); + let client = Client::new(peerid, client_tx); -pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { - match event { - ClientEvent::Listen { addr, sender } => match swarm.listen_on(addr) { - Ok(_) => { - sender.send(Ok(())).unwrap(); - } - Err(e) => { - sender.send(Err(Box::new(e))).unwrap(); - } - }, - } + (client, eventloop) } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 2749f056fc..d450aaadc5 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -18,25 +18,36 @@ impl From for ComposedEvent { } } -pub async fn create_bootstrap(config: ClientConfig) -> Swarm { - // Read a RSA private key from disk, to create a bootstrap node's PeerID. - // let key = identity::Keypair::rsa_from_pkcs8().unwrap(); - // - // - We can read the ClientConfig file, in that method itself. - // - We can even it spawn another task for it. - - if let Some(_addr) = config.listen_addr { - // swarm.listen_on(addr).unwrap(); +pub async fn create_bootstrap(config: ClientConfig) -> (PeerId, Swarm) { + // Generate IDs. + // TODO: Read a RSA private key from disk, to create a bootstrap node's PeerID. + let private_key: &mut [u8] = &mut config.bootstrap_keys.clone()[0]; + let key = identity::Keypair::rsa_from_pkcs8(private_key).unwrap(); + let peerid = PeerId::from_public_key(key.public()); + + let mut swarm = create_swarm(peerid.clone(), key); + + if let Some(addr) = config.listen_addr { + swarm.listen_on(addr).unwrap(); } - todo!() + (peerid, swarm) } -pub async fn create_node(config: ClientConfig) -> Swarm { +pub async fn create_node(config: ClientConfig) -> (PeerId, Swarm) { // Generate IDs. let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); + let mut swarm = create_swarm(peerid.clone(), key); + + // Connect to bootstrap nodes. + dial_bootstrap(&mut swarm, config.bootstrap_nodes); + + (peerid, swarm) +} + +fn create_swarm(peerid: PeerId, key: identity::Keypair) -> Swarm { // Generate NOISE authentication keys. let auth_keys = Keypair::::new().into_authentic(&key).unwrap(); @@ -48,19 +59,14 @@ pub async fn create_node(config: ClientConfig) -> Swarm { .boxed(); let behaviour = ComposedBehaviour { - kademlia: Kademlia::new(peerid.clone(), MemoryStore::new(peerid.clone())), + kademlia: Kademlia::new(peerid, MemoryStore::new(peerid)), }; - let mut swarm = SwarmBuilder::new(transport, behaviour, peerid.clone()) + SwarmBuilder::new(transport, behaviour, peerid) .executor(Box::new(|fut| { tokio::spawn(fut); })) - .build(); - - // Connect to bootstrap nodes. - dial_bootstrap(&mut swarm, config.bootstrap_nodes); - - swarm + .build() } fn dial_bootstrap(swarm: &mut Swarm, nodes: Vec<(Multiaddr, PeerId)>) { diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 78fdf729f3..8c9e6b3680 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -1,4 +1,4 @@ -use super::client::{handle_client_event, ClientEvent}; +use super::client::{Client, ClientEvent}; use super::core::{ComposedBehaviour, ComposedEvent}; use super::*; @@ -30,7 +30,7 @@ impl EventLoop { // NOTE: We have to put the handle client event method in the EventLoop impl because it // needs access to the swarm. fn handle_event(&mut self, event: ClientEvent) { - handle_client_event(&mut self.swarm, event) + Client::handle_client_event(&mut self.swarm, event) } async fn handle_network_event(&mut self, event: SwarmEvent) { diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 5037937cb5..513cbab38e 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -74,6 +74,12 @@ pub(crate) async fn farm( panic!("Identity not found, please create it first using plot command"); } + info!("Opening existing keypair"); + let keypair = + Keypair::from_bytes(&fs::read(identity_file)?).map_err(|error| error.to_string())?; + let public_key_hash = crypto::hash_public_key(&keypair.public); + let ctx = schnorrkel::context::signing_context(SIGNING_CONTEXT); + // When a farmer starts, it should start a libp2p peer, as well. // The peer will connect to a given bootstrap nodes and seek other peers. // 1. Create the swarm with the peer in it. @@ -104,12 +110,6 @@ pub(crate) async fn farm( .await; } - info!("Opening existing keypair"); - let keypair = - Keypair::from_bytes(&fs::read(identity_file)?).map_err(|error| error.to_string())?; - let public_key_hash = crypto::hash_public_key(&keypair.public); - let ctx = schnorrkel::context::signing_context(SIGNING_CONTEXT); - info!("Opening plot"); let plot = Plot::open_or_create(&path.into()).await?; From cbec847bbe3e9b8e5f1b367a496cf23799084356 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 00:29:31 +0530 Subject: [PATCH 19/52] Bootstrap implementation, Part 5. 1. Add `ClientEvent` and `NetworkEvent` for `Bootstrap`. 2. Add flag for inputting bootstrap nodes through the CLI. --- .../spartan-farmer/src/commands/dht/client.rs | 29 +++++++++++++++++-- .../spartan-farmer/src/commands/dht/core.rs | 26 +++++++++++------ .../src/commands/dht/eventloop.rs | 15 ++++++++++ crates/spartan-farmer/src/commands/farm.rs | 9 ++++-- crates/spartan-farmer/src/main.rs | 9 ++++-- 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 857421e7cd..3a8c8e9145 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -2,8 +2,7 @@ use super::core::{create_bootstrap, create_node, ComposedBehaviour}; use super::eventloop::EventLoop; use super::*; -#[derive(Debug)] - +#[derive(Copy, Clone, Debug)] pub enum ClientType { // Bootstrap node. It uses the following fields from `ClientConfig`: // 1. `bootstrap_keys`: Private keys/private key location to create bootstrap node peerId. @@ -21,10 +20,14 @@ pub enum ClientEvent { addr: Multiaddr, sender: oneshot::Sender, }, + // Bootstrap, look for the closest peers. + Bootstrap { + sender: oneshot::Sender, + }, } pub struct ClientConfig { - pub bootstrap_nodes: Vec<(Multiaddr, PeerId)>, + pub bootstrap_nodes: Vec, // Vec<(Multiaddr, PeerId)>, pub bootstrap_keys: Vec>, pub client_type: ClientType, pub listen_addr: Option, @@ -54,6 +57,18 @@ impl Client { let _ = recv.await.expect("Failed to start listening."); } + pub async fn bootstrap(&mut self) { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::Bootstrap { sender }) + .await + .unwrap(); + + // Check if the Bootstrap was processed, properly. + let _ = recv.await.expect("Failed to bootstrap."); + } + pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { match event { ClientEvent::Listen { addr, sender } => match swarm.listen_on(addr) { @@ -64,6 +79,14 @@ impl Client { sender.send(Err(Box::new(e))).unwrap(); } }, + ClientEvent::Bootstrap { sender } => match swarm.behaviour_mut().kademlia.bootstrap() { + Ok(_) => { + sender.send(Ok(())).unwrap(); + } + Err(e) => { + sender.send(Err(Box::new(e))).unwrap(); + } + }, } } } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index d450aaadc5..0cbf844d2b 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -1,11 +1,12 @@ // Pull imports from the parent module use super::client::ClientConfig; use super::*; +use std::str::FromStr; #[derive(NetworkBehaviour)] #[behaviour(event_process = false, out_event = "ComposedEvent")] pub struct ComposedBehaviour { - kademlia: Kademlia, + pub kademlia: Kademlia, } pub enum ComposedEvent { @@ -20,16 +21,20 @@ impl From for ComposedEvent { pub async fn create_bootstrap(config: ClientConfig) -> (PeerId, Swarm) { // Generate IDs. - // TODO: Read a RSA private key from disk, to create a bootstrap node's PeerID. - let private_key: &mut [u8] = &mut config.bootstrap_keys.clone()[0]; - let key = identity::Keypair::rsa_from_pkcs8(private_key).unwrap(); + // TODO: Read a Ed25519 private key from disk, to create a bootstrap node's PeerID. + // let private_key: &mut [u8] = &mut config.bootstrap_keys.clone()[0]; + + let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); let mut swarm = create_swarm(peerid.clone(), key); - if let Some(addr) = config.listen_addr { - swarm.listen_on(addr).unwrap(); - } + match config.listen_addr { + Some(addr) => swarm.listen_on(addr).unwrap(), + None => swarm + .listen_on("/ip4/0.0.0.0/tcp/0".parse().unwrap()) + .unwrap(), + }; (peerid, swarm) } @@ -69,8 +74,11 @@ fn create_swarm(peerid: PeerId, key: identity::Keypair) -> Swarm, nodes: Vec<(Multiaddr, PeerId)>) { +fn dial_bootstrap(swarm: &mut Swarm, nodes: Vec) { for node in nodes { - swarm.behaviour_mut().kademlia.add_address(&node.1, node.0); + let parts: Vec<&str> = node.split("/p2p/").collect(); + let addr = Multiaddr::from_str(parts[0]).unwrap(); + let peer = PeerId::from_str(parts[1]).unwrap(); + swarm.behaviour_mut().kademlia.add_address(&peer, addr); } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 8c9e6b3680..61c310c199 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -37,6 +37,21 @@ impl EventLoop { match event { SwarmEvent::Behaviour(event) => match event { ComposedEvent::Kademlia(event) => match event { + KademliaEvent::RoutingUpdated { peer, .. } => { + info!("Added new peer to routing table: {:?}", peer) + } + KademliaEvent::OutboundQueryCompleted { id, result, .. } => { + info!("Query ID: {:?}", id); + match result { + libp2p::kad::QueryResult::Bootstrap(result) => match result { + Ok(res) => { + info!("Bootstrapping finished successfully: {:?}", res.peer) + } + Err(e) => info!("{:?}", e), + }, + _ => {} + } + } _ => {} }, }, diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 513cbab38e..b0994ccae5 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -60,6 +60,7 @@ struct SlotInfo { /// address. pub(crate) async fn farm( bootstrap: bool, + bootstrap_node: Vec, path: PathBuf, ws_server: &str, ) -> Result<(), Box> { @@ -86,13 +87,14 @@ pub(crate) async fn farm( // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. let node_type = if bootstrap { + info!("I'm a bootstrap node."); ClientType::Bootstrap } else { ClientType::Normal }; let config = ClientConfig { - bootstrap_nodes: Vec::default(), + bootstrap_nodes: bootstrap_node, bootstrap_keys: Vec::default(), client_type: node_type, listen_addr: None, @@ -102,13 +104,16 @@ pub(crate) async fn farm( let (mut dht_client, dht_eventloop) = dht::create_connection(config).await; tokio::spawn(async move { dht_eventloop.run().await }); + info!("My Peer ID is: {:?}", dht_client.peerid); // For bootstrap nodes, we set the listening address through the `ClientConfig`. if !bootstrap { dht_client .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) .await; - } + }; + + dht_client.bootstrap().await; info!("Opening plot"); let plot = Plot::open_or_create(&path.into()).await?; diff --git a/crates/spartan-farmer/src/main.rs b/crates/spartan-farmer/src/main.rs index 84e48343b5..33b1e6a511 100644 --- a/crates/spartan-farmer/src/main.rs +++ b/crates/spartan-farmer/src/main.rs @@ -64,7 +64,11 @@ enum Command { custom_path: Option, #[clap(long, default_value = "ws://127.0.0.1:9944")] ws_server: String, - #[clap(short, long)] + /// List of bootstrap nodes to connect to with. + #[clap(long)] + bootstrap_node: Vec, + /// Am I a bootstrap node? + #[clap(long)] bootstrap: bool, }, } @@ -102,11 +106,12 @@ fn main() { custom_path, ws_server, bootstrap, + bootstrap_node, } => { let path = utils::get_path(custom_path); let runtime = Runtime::new().unwrap(); runtime - .block_on(commands::farm(bootstrap, path, &ws_server)) + .block_on(commands::farm(bootstrap, bootstrap_node, path, &ws_server)) .unwrap(); } } From d71ced55174ad1414113bb474194b9ea5cb63f02 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 01:06:43 +0530 Subject: [PATCH 20/52] Trash commit. Please, Ignore --- .../src/commands/dht/eventloop.rs | 70 +++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 61c310c199..82e1eea82f 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -21,7 +21,7 @@ impl EventLoop { client_event = self.client_rx.next() => self.handle_event(client_event.unwrap()), network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, - None => break, + None => {}, } } } @@ -49,16 +49,78 @@ impl EventLoop { } Err(e) => info!("{:?}", e), }, - _ => {} + libp2p::kad::QueryResult::GetClosestPeers(_) => todo!(), + libp2p::kad::QueryResult::GetProviders(_) => todo!(), + libp2p::kad::QueryResult::StartProviding(_) => todo!(), + libp2p::kad::QueryResult::RepublishProvider(_) => todo!(), + libp2p::kad::QueryResult::GetRecord(_) => todo!(), + libp2p::kad::QueryResult::PutRecord(_) => todo!(), + libp2p::kad::QueryResult::RepublishRecord(_) => todo!(), } } - _ => {} + KademliaEvent::InboundRequestServed { request: _ } => todo!(), + KademliaEvent::UnroutablePeer { peer: _ } => todo!(), + KademliaEvent::RoutablePeer { + peer: _, + address: _, + } => todo!(), + KademliaEvent::PendingRoutablePeer { + peer: _, + address: _, + } => todo!(), }, }, SwarmEvent::NewListenAddr { address, .. } => { info!("Farmer is listening to K-DHT on: {:?}", address) } - _ => {} + SwarmEvent::ConnectionEstablished { + peer_id, + endpoint: _, + num_established: _, + } => info!("Connected to new peer: {:?}", peer_id), + SwarmEvent::ConnectionClosed { + peer_id: _, + endpoint: _, + num_established: _, + cause: _, + } => todo!(), + SwarmEvent::IncomingConnection { + local_addr: _, + send_back_addr: _, + } => todo!(), + SwarmEvent::IncomingConnectionError { + local_addr: _, + send_back_addr: _, + error: _, + } => todo!(), + SwarmEvent::BannedPeer { + peer_id: _, + endpoint: _, + } => todo!(), + SwarmEvent::UnreachableAddr { + peer_id: _, + address: _, + error: _, + attempts_remaining: _, + } => todo!(), + SwarmEvent::UnknownPeerUnreachableAddr { + address: _, + error: _, + } => todo!(), + SwarmEvent::ExpiredListenAddr { + listener_id: _, + address: _, + } => todo!(), + SwarmEvent::ListenerClosed { + listener_id: _, + addresses: _, + reason: _, + } => todo!(), + SwarmEvent::ListenerError { + listener_id: _, + error: _, + } => todo!(), + SwarmEvent::Dialing(_) => todo!(), } } } From ebbe17b25817994cce219f56ab88221126e7f015 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 01:13:20 +0530 Subject: [PATCH 21/52] Trash commit 2 --- .../src/commands/dht/eventloop.rs | 63 +------------------ 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 82e1eea82f..18a57a3af8 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -49,25 +49,10 @@ impl EventLoop { } Err(e) => info!("{:?}", e), }, - libp2p::kad::QueryResult::GetClosestPeers(_) => todo!(), - libp2p::kad::QueryResult::GetProviders(_) => todo!(), - libp2p::kad::QueryResult::StartProviding(_) => todo!(), - libp2p::kad::QueryResult::RepublishProvider(_) => todo!(), - libp2p::kad::QueryResult::GetRecord(_) => todo!(), - libp2p::kad::QueryResult::PutRecord(_) => todo!(), - libp2p::kad::QueryResult::RepublishRecord(_) => todo!(), + _ => {} } } - KademliaEvent::InboundRequestServed { request: _ } => todo!(), - KademliaEvent::UnroutablePeer { peer: _ } => todo!(), - KademliaEvent::RoutablePeer { - peer: _, - address: _, - } => todo!(), - KademliaEvent::PendingRoutablePeer { - peer: _, - address: _, - } => todo!(), + _ => {} }, }, SwarmEvent::NewListenAddr { address, .. } => { @@ -78,49 +63,7 @@ impl EventLoop { endpoint: _, num_established: _, } => info!("Connected to new peer: {:?}", peer_id), - SwarmEvent::ConnectionClosed { - peer_id: _, - endpoint: _, - num_established: _, - cause: _, - } => todo!(), - SwarmEvent::IncomingConnection { - local_addr: _, - send_back_addr: _, - } => todo!(), - SwarmEvent::IncomingConnectionError { - local_addr: _, - send_back_addr: _, - error: _, - } => todo!(), - SwarmEvent::BannedPeer { - peer_id: _, - endpoint: _, - } => todo!(), - SwarmEvent::UnreachableAddr { - peer_id: _, - address: _, - error: _, - attempts_remaining: _, - } => todo!(), - SwarmEvent::UnknownPeerUnreachableAddr { - address: _, - error: _, - } => todo!(), - SwarmEvent::ExpiredListenAddr { - listener_id: _, - address: _, - } => todo!(), - SwarmEvent::ListenerClosed { - listener_id: _, - addresses: _, - reason: _, - } => todo!(), - SwarmEvent::ListenerError { - listener_id: _, - error: _, - } => todo!(), - SwarmEvent::Dialing(_) => todo!(), + _ => {} } } } From c70b67959cc2685ef0626727d08f202bcac9142e Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 02:13:08 +0530 Subject: [PATCH 22/52] Add peers to routing table, whenever we come across a new one --- crates/spartan-farmer/src/commands/dht/eventloop.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 18a57a3af8..384a8a215f 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -21,7 +21,7 @@ impl EventLoop { client_event = self.client_rx.next() => self.handle_event(client_event.unwrap()), network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, - None => {}, + None => break, } } } @@ -52,6 +52,12 @@ impl EventLoop { _ => {} } } + KademliaEvent::RoutablePeer { peer, address } => { + self.swarm + .behaviour_mut() + .kademlia + .add_address(&peer, address); + } _ => {} }, }, From 66be40e484cb74571e0ff9d580005e69be4c4003 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 02:38:52 +0530 Subject: [PATCH 23/52] Clippy fixes. --- .gitignore | 3 +++ crates/spartan-farmer/src/commands/dht/core.rs | 4 ++-- crates/spartan-farmer/src/commands/dht/eventloop.rs | 7 +++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index f419dc7baa..778e1617eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ +.gitignore /.idea /target +**/rustfmt.toml +.vscode/ diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 0cbf844d2b..d5bbe5921b 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -27,7 +27,7 @@ pub async fn create_bootstrap(config: ClientConfig) -> (PeerId, Swarm swarm.listen_on(addr).unwrap(), @@ -44,7 +44,7 @@ pub async fn create_node(config: ClientConfig) -> (PeerId, Swarm { info!("Query ID: {:?}", id); - match result { - libp2p::kad::QueryResult::Bootstrap(result) => match result { + if let libp2p::kad::QueryResult::Bootstrap(result) = result { + match result { Ok(res) => { info!("Bootstrapping finished successfully: {:?}", res.peer) } Err(e) => info!("{:?}", e), - }, - _ => {} + } } } KademliaEvent::RoutablePeer { peer, address } => { From 609bb25586cb68e2d639c1368aa7c3fd9040c38c Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 11:24:51 +0530 Subject: [PATCH 24/52] Remove `create_bootstrap` method as we don't need it. --- .../spartan-farmer/src/commands/dht/client.rs | 6 ++--- .../spartan-farmer/src/commands/dht/core.rs | 23 ++++--------------- crates/spartan-farmer/src/commands/farm.rs | 17 +++++++------- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 3a8c8e9145..6d05663aa7 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,4 +1,4 @@ -use super::core::{create_bootstrap, create_node, ComposedBehaviour}; +use super::core::{create_node, ComposedBehaviour}; use super::eventloop::EventLoop; use super::*; @@ -92,11 +92,11 @@ impl Client { } // This method will construct a new Swarm and EventLoop object. -pub async fn create_connection(config: ClientConfig) -> (Client, EventLoop) { +pub async fn create_connection(config: &ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); let (peerid, swarm) = match config.client_type { - ClientType::Bootstrap => create_bootstrap(config).await, + ClientType::Bootstrap => create_node(config).await, ClientType::Normal => create_node(config).await, }; diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index d5bbe5921b..0584d6de76 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -19,35 +19,22 @@ impl From for ComposedEvent { } } -pub async fn create_bootstrap(config: ClientConfig) -> (PeerId, Swarm) { +pub async fn create_node(config: &ClientConfig) -> (PeerId, Swarm) { // Generate IDs. - // TODO: Read a Ed25519 private key from disk, to create a bootstrap node's PeerID. - // let private_key: &mut [u8] = &mut config.bootstrap_keys.clone()[0]; - let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); let mut swarm = create_swarm(peerid, key); - match config.listen_addr { - Some(addr) => swarm.listen_on(addr).unwrap(), + match &config.listen_addr { + Some(addr) => swarm.listen_on(addr.clone()).unwrap(), None => swarm .listen_on("/ip4/0.0.0.0/tcp/0".parse().unwrap()) .unwrap(), }; - (peerid, swarm) -} - -pub async fn create_node(config: ClientConfig) -> (PeerId, Swarm) { - // Generate IDs. - let key = identity::Keypair::generate_ed25519(); - let peerid = PeerId::from_public_key(key.public()); - - let mut swarm = create_swarm(peerid, key); - // Connect to bootstrap nodes. - dial_bootstrap(&mut swarm, config.bootstrap_nodes); + dial_bootstrap(&mut swarm, &config.bootstrap_nodes); (peerid, swarm) } @@ -74,7 +61,7 @@ fn create_swarm(peerid: PeerId, key: identity::Keypair) -> Swarm, nodes: Vec) { +fn dial_bootstrap(swarm: &mut Swarm, nodes: &Vec) { for node in nodes { let parts: Vec<&str> = node.split("/p2p/").collect(); let addr = Multiaddr::from_str(parts[0]).unwrap(); diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index b0994ccae5..49d2fddea0 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -86,28 +86,27 @@ pub(crate) async fn farm( // 1. Create the swarm with the peer in it. // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. - let node_type = if bootstrap { - info!("I'm a bootstrap node."); - ClientType::Bootstrap - } else { - ClientType::Normal - }; let config = ClientConfig { bootstrap_nodes: bootstrap_node, bootstrap_keys: Vec::default(), - client_type: node_type, + client_type: if bootstrap { + info!("I'm a bootstrap node."); + ClientType::Bootstrap + } else { + ClientType::Normal + }, listen_addr: None, }; info!("Connecting to DHT"); - let (mut dht_client, dht_eventloop) = dht::create_connection(config).await; + let (mut dht_client, dht_eventloop) = dht::create_connection(&config).await; tokio::spawn(async move { dht_eventloop.run().await }); info!("My Peer ID is: {:?}", dht_client.peerid); // For bootstrap nodes, we set the listening address through the `ClientConfig`. - if !bootstrap { + if config.listen_addr.is_none() { dht_client .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) .await; From 37a8f4ce43b96f502cce2be7c0734882ab49fcc1 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 11:43:04 +0530 Subject: [PATCH 25/52] Add flag for `listen_addr`. --- .../spartan-farmer/src/commands/dht/core.rs | 9 ++--- crates/spartan-farmer/src/commands/farm.rs | 39 ++++++++++++------- crates/spartan-farmer/src/main.rs | 20 +++++++--- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 0584d6de76..4592edf653 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -26,12 +26,9 @@ pub async fn create_node(config: &ClientConfig) -> (PeerId, Swarm swarm.listen_on(addr.clone()).unwrap(), - None => swarm - .listen_on("/ip4/0.0.0.0/tcp/0".parse().unwrap()) - .unwrap(), - }; + if let Some(addr) = &config.listen_addr { + swarm.listen_on(addr.clone()).unwrap(); + } // Connect to bootstrap nodes. dial_bootstrap(&mut swarm, &config.bootstrap_nodes); diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 49d2fddea0..0c1cd44cca 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -1,3 +1,7 @@ +use super::dht::{ + client as dht, + client::{ClientConfig, ClientType}, +}; use crate::plot::Plot; use crate::{crypto, Salt, Tag, PRIME_SIZE_BYTES, SIGNING_CONTEXT}; use async_std::task; @@ -5,6 +9,7 @@ use futures::channel::oneshot; use jsonrpsee::ws_client::traits::{Client, SubscriptionClient}; use jsonrpsee::ws_client::v2::params::JsonRpcParams; use jsonrpsee::ws_client::{Subscription, WsClientBuilder}; +use libp2p::Multiaddr; use log::{debug, error, info, trace, warn}; use ring::digest; use schnorrkel::Keypair; @@ -12,13 +17,9 @@ use serde::{Deserialize, Serialize}; use std::convert::TryInto; use std::fs; use std::path::PathBuf; +use std::str::FromStr; use std::time::Instant; -use super::dht::{ - client as dht, - client::{ClientConfig, ClientType}, -}; - type SlotNumber = u64; #[derive(Debug, Serialize)] @@ -59,8 +60,9 @@ struct SlotInfo { /// Start farming by using plot in specified path and connecting to WebSocket server at specified /// address. pub(crate) async fn farm( + listen_addr: String, bootstrap: bool, - bootstrap_node: Vec, + bootstrap_nodes: Vec, path: PathBuf, ws_server: &str, ) -> Result<(), Box> { @@ -87,16 +89,24 @@ pub(crate) async fn farm( // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. + let client_type = if bootstrap { + info!("I'm a bootstrap node."); + ClientType::Bootstrap + } else { + ClientType::Normal + }; + + let listen_addr = if listen_addr.is_empty() { + None + } else { + Some(Multiaddr::from_str(&listen_addr)?) + }; + let config = ClientConfig { - bootstrap_nodes: bootstrap_node, + bootstrap_nodes, bootstrap_keys: Vec::default(), - client_type: if bootstrap { - info!("I'm a bootstrap node."); - ClientType::Bootstrap - } else { - ClientType::Normal - }, - listen_addr: None, + client_type, + listen_addr, }; info!("Connecting to DHT"); @@ -105,7 +115,6 @@ pub(crate) async fn farm( tokio::spawn(async move { dht_eventloop.run().await }); info!("My Peer ID is: {:?}", dht_client.peerid); - // For bootstrap nodes, we set the listening address through the `ClientConfig`. if config.listen_addr.is_none() { dht_client .start_listening("/ip4/0.0.0.0/tcp/0".parse()?) diff --git a/crates/spartan-farmer/src/main.rs b/crates/spartan-farmer/src/main.rs index 33b1e6a511..ae1d1d5141 100644 --- a/crates/spartan-farmer/src/main.rs +++ b/crates/spartan-farmer/src/main.rs @@ -66,10 +66,13 @@ enum Command { ws_server: String, /// List of bootstrap nodes to connect to with. #[clap(long)] - bootstrap_node: Vec, - /// Am I a bootstrap node? - #[clap(long)] + bootstrap_nodes: Vec, + /// Set this peer as bootstrap node. + #[clap(long, short)] bootstrap: bool, + /// Listening address for P2P peer. + #[clap(long)] + listen_addr: String, }, } @@ -106,12 +109,19 @@ fn main() { custom_path, ws_server, bootstrap, - bootstrap_node, + bootstrap_nodes, + listen_addr, } => { let path = utils::get_path(custom_path); let runtime = Runtime::new().unwrap(); runtime - .block_on(commands::farm(bootstrap, bootstrap_node, path, &ws_server)) + .block_on(commands::farm( + listen_addr, + bootstrap, + bootstrap_nodes, + path, + &ws_server, + )) .unwrap(); } } From e035caf4e6fdae891960de7ab63dfb6432d31e1e Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 11:54:38 +0530 Subject: [PATCH 26/52] Better comments. --- crates/spartan-farmer/src/commands/dht/client.rs | 1 + crates/spartan-farmer/src/commands/dht/eventloop.rs | 7 +++---- crates/spartan-farmer/src/commands/farm.rs | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 6d05663aa7..80cafbaab5 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -57,6 +57,7 @@ impl Client { let _ = recv.await.expect("Failed to start listening."); } + // Sync with other peers on the DHT. pub async fn bootstrap(&mut self) { let (sender, recv) = oneshot::channel(); diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 9862d8acea..0172105a1a 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -9,12 +9,12 @@ pub struct EventLoop { } impl EventLoop { - /// Create new event loop + // Create new event loop pub fn new(swarm: Swarm, client_rx: Receiver) -> Self { EventLoop { swarm, client_rx } } - /// Run event loop. We will use this method to spawn the event loop in a background task. + // Run event loop. We will use this method to spawn the event loop in a background task. pub async fn run(mut self) { loop { futures::select! { @@ -27,8 +27,7 @@ impl EventLoop { } } - // NOTE: We have to put the handle client event method in the EventLoop impl because it - // needs access to the swarm. + // The Client will send events to EventLoop using this method. fn handle_event(&mut self, event: ClientEvent) { Client::handle_client_event(&mut self.swarm, event) } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 0c1cd44cca..32ff8565ed 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -121,6 +121,9 @@ pub(crate) async fn farm( .await; }; + // TODO: Ideally, this should be performed periodically after a fixed time interval. + // Peer Discovery needs to be a repetitive process, to make sure no peers carry stale + // information. dht_client.bootstrap().await; info!("Opening plot"); From f8a8f31948c13e1bfa4bb3a17c893c0f0f7b1077 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 12:35:42 +0530 Subject: [PATCH 27/52] Add `Result` return type to `handle_client_event`. --- .../spartan-farmer/src/commands/dht/client.rs | 29 +++++++++---------- .../src/commands/dht/eventloop.rs | 11 +++++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 80cafbaab5..6d0431052d 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,4 +1,4 @@ -use super::core::{create_node, ComposedBehaviour}; +use super::core::create_node; use super::eventloop::EventLoop; use super::*; @@ -70,24 +70,21 @@ impl Client { let _ = recv.await.expect("Failed to bootstrap."); } - pub fn handle_client_event(swarm: &mut Swarm, event: ClientEvent) { + pub fn handle_client_event( + eventloop: &mut EventLoop, + event: ClientEvent, + ) -> Result<(), OneshotType> { match event { - ClientEvent::Listen { addr, sender } => match swarm.listen_on(addr) { - Ok(_) => { - sender.send(Ok(())).unwrap(); - } - Err(e) => { - sender.send(Err(Box::new(e))).unwrap(); - } + ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { + Ok(_) => sender.send(Ok(())), + Err(e) => sender.send(Err(Box::new(e))), }, - ClientEvent::Bootstrap { sender } => match swarm.behaviour_mut().kademlia.bootstrap() { - Ok(_) => { - sender.send(Ok(())).unwrap(); + ClientEvent::Bootstrap { sender } => { + match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { + Ok(_) => sender.send(Ok(())), + Err(e) => sender.send(Err(Box::new(e))), } - Err(e) => { - sender.send(Err(Box::new(e))).unwrap(); - } - }, + } } } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 0172105a1a..f0295c348b 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -3,7 +3,7 @@ use super::core::{ComposedBehaviour, ComposedEvent}; use super::*; pub struct EventLoop { - swarm: Swarm, + pub swarm: Swarm, // Channel to receive events from Client. client_rx: Receiver, } @@ -18,7 +18,10 @@ impl EventLoop { pub async fn run(mut self) { loop { futures::select! { - client_event = self.client_rx.next() => self.handle_event(client_event.unwrap()), + client_event = self.client_rx.next() => match client_event { + Some(event) => self.handle_event(event), + None => {}, + }, network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, None => break, @@ -29,7 +32,9 @@ impl EventLoop { // The Client will send events to EventLoop using this method. fn handle_event(&mut self, event: ClientEvent) { - Client::handle_client_event(&mut self.swarm, event) + if let Err(e) = Client::handle_client_event(self, event) { + info!("{:?}", e) + } } async fn handle_network_event(&mut self, event: SwarmEvent) { From 7bc8bf6f2382ae9ef5ea1f6a603960d6614f32da Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 12:39:47 +0530 Subject: [PATCH 28/52] Change type for `listen_addr` in `main.rs` CLAP arguments --- crates/spartan-farmer/src/commands/dht/client.rs | 1 - crates/spartan-farmer/src/commands/farm.rs | 12 ++---------- crates/spartan-farmer/src/main.rs | 3 ++- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 6d0431052d..d4be4459a3 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -28,7 +28,6 @@ pub enum ClientEvent { pub struct ClientConfig { pub bootstrap_nodes: Vec, // Vec<(Multiaddr, PeerId)>, - pub bootstrap_keys: Vec>, pub client_type: ClientType, pub listen_addr: Option, } diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 32ff8565ed..7982a7381d 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -17,7 +17,6 @@ use serde::{Deserialize, Serialize}; use std::convert::TryInto; use std::fs; use std::path::PathBuf; -use std::str::FromStr; use std::time::Instant; type SlotNumber = u64; @@ -60,7 +59,7 @@ struct SlotInfo { /// Start farming by using plot in specified path and connecting to WebSocket server at specified /// address. pub(crate) async fn farm( - listen_addr: String, + listen_addr: Option, bootstrap: bool, bootstrap_nodes: Vec, path: PathBuf, @@ -84,7 +83,7 @@ pub(crate) async fn farm( let ctx = schnorrkel::context::signing_context(SIGNING_CONTEXT); // When a farmer starts, it should start a libp2p peer, as well. - // The peer will connect to a given bootstrap nodes and seek other peers. + // The peer will connect to given bootstrap nodes and seek other peers. // 1. Create the swarm with the peer in it. // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. @@ -96,15 +95,8 @@ pub(crate) async fn farm( ClientType::Normal }; - let listen_addr = if listen_addr.is_empty() { - None - } else { - Some(Multiaddr::from_str(&listen_addr)?) - }; - let config = ClientConfig { bootstrap_nodes, - bootstrap_keys: Vec::default(), client_type, listen_addr, }; diff --git a/crates/spartan-farmer/src/main.rs b/crates/spartan-farmer/src/main.rs index ae1d1d5141..8a48a4791d 100644 --- a/crates/spartan-farmer/src/main.rs +++ b/crates/spartan-farmer/src/main.rs @@ -23,6 +23,7 @@ mod utils; use async_std::task; use clap::{Clap, ValueHint}; use env_logger::Env; +use libp2p::Multiaddr; use log::info; use std::fs; use std::path::PathBuf; @@ -72,7 +73,7 @@ enum Command { bootstrap: bool, /// Listening address for P2P peer. #[clap(long)] - listen_addr: String, + listen_addr: Option, }, } From 668ada94a83411b93f4b39e25310c6ac8b73e4d1 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 12:50:18 +0530 Subject: [PATCH 29/52] Format dependencies in a prettier way. --- crates/spartan-farmer/src/commands/dht/client.rs | 3 +-- crates/spartan-farmer/src/commands/dht/eventloop.rs | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index d4be4459a3..a8f4c97bcd 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,6 +1,5 @@ -use super::core::create_node; -use super::eventloop::EventLoop; use super::*; +use super::{core::create_node, eventloop::EventLoop}; #[derive(Copy, Clone, Debug)] pub enum ClientType { diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index f0295c348b..576a40ce83 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -1,6 +1,8 @@ -use super::client::{Client, ClientEvent}; -use super::core::{ComposedBehaviour, ComposedEvent}; use super::*; +use super::{ + client::{Client, ClientEvent}, + core::{ComposedBehaviour, ComposedEvent}, +}; pub struct EventLoop { pub swarm: Swarm, From 709dfbecdc57887b865166ffc2c448aaad8d7fe0 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 12:53:45 +0530 Subject: [PATCH 30/52] Remove unnecessary match statements in `handle_network_event` --- .../spartan-farmer/src/commands/dht/client.rs | 2 +- .../src/commands/dht/eventloop.rs | 49 +++++++++---------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index a8f4c97bcd..6746a6ee0e 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -79,7 +79,7 @@ impl Client { }, ClientEvent::Bootstrap { sender } => { match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - Ok(_) => sender.send(Ok(())), + Ok(_qid) => sender.send(Ok(())), Err(e) => sender.send(Err(Box::new(e))), } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 576a40ce83..5142b44034 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -39,41 +39,38 @@ impl EventLoop { } } + // Handle network events. async fn handle_network_event(&mut self, event: SwarmEvent) { match event { - SwarmEvent::Behaviour(event) => match event { - ComposedEvent::Kademlia(event) => match event { - KademliaEvent::RoutingUpdated { peer, .. } => { - info!("Added new peer to routing table: {:?}", peer) - } - KademliaEvent::OutboundQueryCompleted { id, result, .. } => { - info!("Query ID: {:?}", id); - if let libp2p::kad::QueryResult::Bootstrap(result) = result { - match result { - Ok(res) => { - info!("Bootstrapping finished successfully: {:?}", res.peer) - } - Err(e) => info!("{:?}", e), + SwarmEvent::Behaviour(ComposedEvent::Kademlia(event)) => match event { + KademliaEvent::RoutingUpdated { peer, .. } => { + info!("Added new peer to routing table: {:?}", peer) + } + KademliaEvent::OutboundQueryCompleted { id, result, .. } => { + info!("Query ID: {:?}", id); + if let libp2p::kad::QueryResult::Bootstrap(result) = result { + match result { + Ok(res) => { + info!("Bootstrapping finished successfully: {:?}", res.peer) } + Err(e) => info!("{:?}", e), } } - KademliaEvent::RoutablePeer { peer, address } => { - self.swarm - .behaviour_mut() - .kademlia - .add_address(&peer, address); - } - _ => {} - }, + } + KademliaEvent::RoutablePeer { peer, address } => { + self.swarm + .behaviour_mut() + .kademlia + .add_address(&peer, address); + } + _ => {} }, SwarmEvent::NewListenAddr { address, .. } => { info!("Farmer is listening to K-DHT on: {:?}", address) } - SwarmEvent::ConnectionEstablished { - peer_id, - endpoint: _, - num_established: _, - } => info!("Connected to new peer: {:?}", peer_id), + SwarmEvent::ConnectionEstablished { peer_id, .. } => { + info!("Connected to new peer: {:?}", peer_id) + } _ => {} } } From 57292e7890033b159c8d4ee2aa01417519995498 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 14:31:10 +0530 Subject: [PATCH 31/52] Add `test` module. --- crates/spartan-farmer/src/commands.rs | 2 +- crates/spartan-farmer/src/commands/dht.rs | 4 ++++ crates/spartan-farmer/src/commands/dht/client.rs | 2 ++ crates/spartan-farmer/src/commands/dht/test.rs | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 crates/spartan-farmer/src/commands/dht/test.rs diff --git a/crates/spartan-farmer/src/commands.rs b/crates/spartan-farmer/src/commands.rs index b316d4114a..7174e9cede 100644 --- a/crates/spartan-farmer/src/commands.rs +++ b/crates/spartan-farmer/src/commands.rs @@ -1,4 +1,4 @@ -pub mod dht; +mod dht; mod farm; mod plot; diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 9c7b00a7bf..e8b9f46274 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -34,3 +34,7 @@ mod core; // EventLoop which actually processes libp2p SwarmEvents. The Client API interacts with the // EventLoop to transfer and receieve data. mod eventloop; + +// DHT related tests. +#[cfg(test)] +mod test; diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 6746a6ee0e..142f0ab673 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -44,6 +44,8 @@ impl Client { // Set listening address for a particular Normal node. pub async fn start_listening(&mut self, addr: Multiaddr) { + // The oneshot channel helps us to pass error messages related to + // SwarmEvent/KademliaEvent. let (sender, recv) = oneshot::channel(); self.client_tx diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/commands/dht/test.rs new file mode 100644 index 0000000000..4563e55b7b --- /dev/null +++ b/crates/spartan-farmer/src/commands/dht/test.rs @@ -0,0 +1 @@ +use super::*; From 44bbd8a67a831cbbad80d374ccc4a253b4138ee1 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 14:41:49 +0530 Subject: [PATCH 32/52] Clippy fixes. --- crates/spartan-farmer/src/commands/dht/eventloop.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 5142b44034..5ca59cea84 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -20,9 +20,8 @@ impl EventLoop { pub async fn run(mut self) { loop { futures::select! { - client_event = self.client_rx.next() => match client_event { - Some(event) => self.handle_event(event), - None => {}, + client_event = self.client_rx.next() => if let Some(event) = client_event { + self.handle_event(event) }, network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, From cc9829fbf3c7b20bca96ee9b789f9a225d9decde Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 15:25:38 +0530 Subject: [PATCH 33/52] Make `create_connection` method synchronous from async --- crates/spartan-farmer/src/commands/dht.rs | 1 + crates/spartan-farmer/src/commands/dht/client.rs | 6 +++--- crates/spartan-farmer/src/commands/dht/core.rs | 2 +- crates/spartan-farmer/src/commands/farm.rs | 2 +- rustfmt.toml | 4 ---- 5 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 rustfmt.toml diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index e8b9f46274..25d7b147bf 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -12,6 +12,7 @@ use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; use libp2p::swarm::{SwarmBuilder, SwarmEvent}; use libp2p::tcp::TokioTcpConfig; use libp2p::{Multiaddr, PeerId, Swarm}; +use std::str::FromStr; // Stuff needed to set up channels between Client API task and EventLoop task. use futures::channel::{ diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 142f0ab673..53248042bd 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -90,12 +90,12 @@ impl Client { } // This method will construct a new Swarm and EventLoop object. -pub async fn create_connection(config: &ClientConfig) -> (Client, EventLoop) { +pub fn create_connection(config: &ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); let (peerid, swarm) = match config.client_type { - ClientType::Bootstrap => create_node(config).await, - ClientType::Normal => create_node(config).await, + ClientType::Bootstrap => create_node(config), + ClientType::Normal => create_node(config), }; let eventloop = EventLoop::new(swarm, client_rx); diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 4592edf653..bebc20a2c0 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -19,7 +19,7 @@ impl From for ComposedEvent { } } -pub async fn create_node(config: &ClientConfig) -> (PeerId, Swarm) { +pub fn create_node(config: &ClientConfig) -> (PeerId, Swarm) { // Generate IDs. let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 7982a7381d..07862e1a33 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -102,7 +102,7 @@ pub(crate) async fn farm( }; info!("Connecting to DHT"); - let (mut dht_client, dht_eventloop) = dht::create_connection(&config).await; + let (mut dht_client, dht_eventloop) = dht::create_connection(&config); tokio::spawn(async move { dht_eventloop.run().await }); info!("My Peer ID is: {:?}", dht_client.peerid); diff --git a/rustfmt.toml b/rustfmt.toml deleted file mode 100644 index 20ad52fa9b..0000000000 --- a/rustfmt.toml +++ /dev/null @@ -1,4 +0,0 @@ -ignore = [ - # This is where Substrate sources with custom formatting are, don't touch them - "/substrate", -] From 1f5357e18245ab193594e35350ddb21fe0f1899f Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 16:37:47 +0530 Subject: [PATCH 34/52] Add `KnownPeers` client event. --- .../spartan-farmer/src/commands/dht/client.rs | 45 +++++++++++++++---- .../src/commands/dht/eventloop.rs | 4 +- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 53248042bd..2cd23deef1 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -23,6 +23,10 @@ pub enum ClientEvent { Bootstrap { sender: oneshot::Sender, }, + // List all known peers. + KnownPeers { + sender: oneshot::Sender>, + }, } pub struct ClientConfig { @@ -42,6 +46,21 @@ impl Client { Client { peerid, client_tx } } + // Returns the list of all the peers the client has in its Routing table. + pub async fn known_peers(&mut self) -> Vec { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::KnownPeers { sender }) + .await; + + let peers = recv + .await + .expect("Failed to retrieve the list of all known peers."); + + peers + } + // Set listening address for a particular Normal node. pub async fn start_listening(&mut self, addr: Multiaddr) { // The oneshot channel helps us to pass error messages related to @@ -70,21 +89,31 @@ impl Client { let _ = recv.await.expect("Failed to bootstrap."); } - pub fn handle_client_event( - eventloop: &mut EventLoop, - event: ClientEvent, - ) -> Result<(), OneshotType> { + pub fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { match event { ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { - Ok(_) => sender.send(Ok(())), - Err(e) => sender.send(Err(Box::new(e))), + Ok(_) => sender.send(Ok(())).unwrap(), + Err(e) => sender.send(Err(Box::new(e))).unwrap(), }, ClientEvent::Bootstrap { sender } => { match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - Ok(_qid) => sender.send(Ok(())), - Err(e) => sender.send(Err(Box::new(e))), + Ok(_qid) => sender.send(Ok(())).unwrap(), + Err(e) => sender.send(Err(Box::new(e))).unwrap(), } } + ClientEvent::KnownPeers { sender } => { + let mut result = Vec::new(); + + for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { + if !bucket.is_empty() { + for record in bucket.iter() { + result.push(*record.node.key.preimage()); + } + } + } + + sender.send(result); + } } } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 5ca59cea84..6903ae14b9 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -33,9 +33,7 @@ impl EventLoop { // The Client will send events to EventLoop using this method. fn handle_event(&mut self, event: ClientEvent) { - if let Err(e) = Client::handle_client_event(self, event) { - info!("{:?}", e) - } + Client::handle_client_event(self, event) } // Handle network events. From 0165b73bfbabe05b6631b8c07dd429bd699dc3b3 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 17:05:55 +0530 Subject: [PATCH 35/52] Add `Dial` Client Event. --- crates/spartan-farmer/src/commands/dht.rs | 1 - .../spartan-farmer/src/commands/dht/client.rs | 37 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index 25d7b147bf..e8b9f46274 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -12,7 +12,6 @@ use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; use libp2p::swarm::{SwarmBuilder, SwarmEvent}; use libp2p::tcp::TokioTcpConfig; use libp2p::{Multiaddr, PeerId, Swarm}; -use std::str::FromStr; // Stuff needed to set up channels between Client API task and EventLoop task. use futures::channel::{ diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 2cd23deef1..ffd5ce623b 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,5 +1,6 @@ use super::*; use super::{core::create_node, eventloop::EventLoop}; +use libp2p::multiaddr::Protocol; #[derive(Copy, Clone, Debug)] pub enum ClientType { @@ -27,6 +28,12 @@ pub enum ClientEvent { KnownPeers { sender: oneshot::Sender>, }, + // Dial another peer. + Dial { + addr: Multiaddr, + peer: PeerId, + sender: oneshot::Sender, + }, } pub struct ClientConfig { @@ -46,13 +53,25 @@ impl Client { Client { peerid, client_tx } } + pub async fn dial(&mut self, peer: PeerId, addr: Multiaddr) { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::Dial { addr, peer, sender }) + .await + .unwrap(); + + let _ = recv.await; + } + // Returns the list of all the peers the client has in its Routing table. pub async fn known_peers(&mut self) -> Vec { let (sender, recv) = oneshot::channel(); self.client_tx .send(ClientEvent::KnownPeers { sender }) - .await; + .await + .unwrap(); let peers = recv .await @@ -112,7 +131,21 @@ impl Client { } } - sender.send(result); + sender.send(result).unwrap(); + } + ClientEvent::Dial { addr, peer, sender } => { + eventloop + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer, addr.clone()); + + eventloop + .swarm + .dial_addr(addr.with(Protocol::P2p(peer.into()))) + .unwrap(); + + sender.send(Ok(())).unwrap(); } } } From 79b40d0a7b15ae72baa11f884f7df025526aca3b Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Tue, 21 Sep 2021 23:33:32 +0530 Subject: [PATCH 36/52] Add `GetRecord` event. --- .../spartan-farmer/src/commands/dht/client.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index ffd5ce623b..4a2e1e4d5a 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -34,6 +34,12 @@ pub enum ClientEvent { peer: PeerId, sender: oneshot::Sender, }, + // NOTE: I'm not sure if this will work without a Provider. + // Get a particular record. + GetRecord { + key: Key, + sender: oneshot::Sender, + }, } pub struct ClientConfig { @@ -53,6 +59,19 @@ impl Client { Client { peerid, client_tx } } + // Get a particular record from K-DHT. + pub async fn get_record(&mut self, key: Key) { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::GetRecord { key, sender }) + .await + .unwrap(); + + let _query_id = recv.await; + } + + // Dial another node using Peer Id and Address. pub async fn dial(&mut self, peer: PeerId, addr: Multiaddr) { let (sender, recv) = oneshot::channel(); @@ -147,6 +166,15 @@ impl Client { sender.send(Ok(())).unwrap(); } + ClientEvent::GetRecord { key, sender } => { + let qid = eventloop + .swarm + .behaviour_mut() + .kademlia + .get_record(&key, Quorum::One); + + sender.send(qid).unwrap(); + } } } } From 02d36d924838d790a917c607a9982db6667f0a1a Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 05:00:37 +0530 Subject: [PATCH 37/52] Add new client events: Random Walk, Listener, Query Result --- crates/spartan-farmer/Cargo.toml | 2 +- crates/spartan-farmer/src/commands/dht.rs | 2 +- .../spartan-farmer/src/commands/dht/client.rs | 133 +++++++++++++----- .../spartan-farmer/src/commands/dht/core.rs | 1 + .../src/commands/dht/eventloop.rs | 40 +++--- 5 files changed, 126 insertions(+), 52 deletions(-) diff --git a/crates/spartan-farmer/Cargo.toml b/crates/spartan-farmer/Cargo.toml index 008840ee46..7e3ea980f6 100644 --- a/crates/spartan-farmer/Cargo.toml +++ b/crates/spartan-farmer/Cargo.toml @@ -30,7 +30,7 @@ serde_json = "1.0.64" schnorrkel = "0.10.1" spartan-codec = "0.1.0" thiserror = "1.0.24" -tokio = "1.11.0" +tokio = { version = "1.11.0", features = [ "rt", "macros", "fs" ] } [dependencies.jsonrpsee] features = ["client"] diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs index e8b9f46274..0fd0825271 100644 --- a/crates/spartan-farmer/src/commands/dht.rs +++ b/crates/spartan-farmer/src/commands/dht.rs @@ -1,5 +1,5 @@ // Stuff for Kademlia -use libp2p::kad::{record::store::MemoryStore, Kademlia, KademliaEvent}; +use libp2p::kad::{record::store::MemoryStore, Kademlia, KademliaEvent, QueryId, QueryResult}; // Stuff for defining composed behaviour use libp2p::NetworkBehaviour; diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 4a2e1e4d5a..2e87b848a8 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -20,9 +20,10 @@ pub enum ClientEvent { addr: Multiaddr, sender: oneshot::Sender, }, - // Bootstrap, look for the closest peers. - Bootstrap { - sender: oneshot::Sender, + // Kademlia Random Walk for Peer Discovery. (GetClosestPeer) + RandomWalk { + key: Option, + sender: oneshot::Sender, }, // List all known peers. KnownPeers { @@ -34,12 +35,21 @@ pub enum ClientEvent { peer: PeerId, sender: oneshot::Sender, }, - // NOTE: I'm not sure if this will work without a Provider. - // Get a particular record. - GetRecord { - key: Key, + // Bootstrap during the initial connection to the DHT. + // NOTE: All the bootstrap nodes must already be connected to the swarm before we can start the + // bootstrap process. + Bootstrap { sender: oneshot::Sender, }, + // Get all listening addresses. + Listeners { + sender: oneshot::Sender>, + }, + // Read Kademlia Query Result. + QueryResult { + qid: QueryId, + sender: oneshot::Sender, + }, } pub struct ClientConfig { @@ -59,16 +69,36 @@ impl Client { Client { peerid, client_tx } } - // Get a particular record from K-DHT. - pub async fn get_record(&mut self, key: Key) { + // Read the Query Result for a specific Kademlia query. + async fn query_result(&mut self, qid: QueryId) -> QueryResult { let (sender, recv) = oneshot::channel(); self.client_tx - .send(ClientEvent::GetRecord { key, sender }) + .send(ClientEvent::QueryResult { qid, sender }) .await .unwrap(); - let _query_id = recv.await; + let result = recv + .await + .expect("Failed to retrieve the list of all known peers."); + + result + } + + // Get the list of all addresses we are listening on. + pub async fn listeners(&mut self) -> Vec { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::Listeners { sender }) + .await + .unwrap(); + + let addrs = recv + .await + .expect("Failed to retrieve the list of all known peers."); + + addrs } // Dial another node using Peer Id and Address. @@ -114,7 +144,20 @@ impl Client { let _ = recv.await.expect("Failed to start listening."); } - // Sync with other peers on the DHT. + // Sync with other peers on the DHT. (GetClosestPeer) + pub async fn random_walk(&mut self, key: Option) { + let (sender, recv) = oneshot::channel(); + + self.client_tx + .send(ClientEvent::RandomWalk { key, sender }) + .await + .unwrap(); + + // Check if the Bootstrap was processed, properly. + let _ = recv.await.expect("Failed to Random Walk."); + } + + // Bootstrap pub async fn bootstrap(&mut self) { let (sender, recv) = oneshot::channel(); @@ -124,7 +167,7 @@ impl Client { .unwrap(); // Check if the Bootstrap was processed, properly. - let _ = recv.await.expect("Failed to bootstrap."); + let _qid = recv.await.expect("Failed to bootstrap."); } pub fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { @@ -134,19 +177,38 @@ impl Client { Err(e) => sender.send(Err(Box::new(e))).unwrap(), }, ClientEvent::Bootstrap { sender } => { - match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - Ok(_qid) => sender.send(Ok(())).unwrap(), - Err(e) => sender.send(Err(Box::new(e))).unwrap(), + if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { + sender.send(qid).unwrap(); } + // match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { + // Ok(_qid) => sender.send(Ok(())).unwrap(), + // Err(e) => sender.send(Err(Box::new(e))).unwrap(), + // } + } + ClientEvent::RandomWalk { sender, key } => { + // NOTE: An interesting fact, that I have noticed is that Kademlia is not + // bidirectional. For example, if Peer 1 adds Peer 2 to its routing table, Peer 2 + // will not add Peer 1 to its routing table. + + let key = match key { + Some(peerid) => peerid, + None => PeerId::random(), + }; + + let qid = eventloop + .swarm + .behaviour_mut() + .kademlia + .get_closest_peers(key); + + sender.send(qid).unwrap(); } ClientEvent::KnownPeers { sender } => { let mut result = Vec::new(); for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { - if !bucket.is_empty() { - for record in bucket.iter() { - result.push(*record.node.key.preimage()); - } + for record in bucket.iter() { + result.push(record.node.key.preimage().clone()); } } @@ -159,21 +221,28 @@ impl Client { .kademlia .add_address(&peer, addr.clone()); - eventloop - .swarm - .dial_addr(addr.with(Protocol::P2p(peer.into()))) - .unwrap(); + // eventloop + // .swarm + // .dial_addr(addr.with(Protocol::P2p(peer.into()))) + // .unwrap(); sender.send(Ok(())).unwrap(); } - ClientEvent::GetRecord { key, sender } => { - let qid = eventloop - .swarm - .behaviour_mut() - .kademlia - .get_record(&key, Quorum::One); - - sender.send(qid).unwrap(); + ClientEvent::Listeners { sender } => { + sender + .send( + eventloop + .swarm + .listeners() + .map(|addr| addr.clone()) + .collect(), + ) + .unwrap(); + } + ClientEvent::QueryResult { qid, sender } => { + sender + .send(eventloop.query_result.remove(&qid).unwrap()) + .unwrap(); } } } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index bebc20a2c0..a0046df391 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -64,5 +64,6 @@ fn dial_bootstrap(swarm: &mut Swarm, nodes: &Vec) { let addr = Multiaddr::from_str(parts[0]).unwrap(); let peer = PeerId::from_str(parts[1]).unwrap(); swarm.behaviour_mut().kademlia.add_address(&peer, addr); + // swarm.dial_addr(Multiaddr::from_str(node).unwrap()).unwrap(); } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index 6903ae14b9..c2fe4430d2 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -3,17 +3,24 @@ use super::{ client::{Client, ClientEvent}, core::{ComposedBehaviour, ComposedEvent}, }; +use std::collections::HashMap; pub struct EventLoop { pub swarm: Swarm, // Channel to receive events from Client. client_rx: Receiver, + // HashMap to send back QueryResults. + pub query_result: HashMap, } impl EventLoop { // Create new event loop pub fn new(swarm: Swarm, client_rx: Receiver) -> Self { - EventLoop { swarm, client_rx } + EventLoop { + swarm, + client_rx, + query_result: HashMap::default(), + } } // Run event loop. We will use this method to spawn the event loop in a background task. @@ -21,7 +28,7 @@ impl EventLoop { loop { futures::select! { client_event = self.client_rx.next() => if let Some(event) = client_event { - self.handle_event(event) + Client::handle_client_event(&mut self, event) }, network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, @@ -31,11 +38,6 @@ impl EventLoop { } } - // The Client will send events to EventLoop using this method. - fn handle_event(&mut self, event: ClientEvent) { - Client::handle_client_event(self, event) - } - // Handle network events. async fn handle_network_event(&mut self, event: SwarmEvent) { match event { @@ -44,21 +46,23 @@ impl EventLoop { info!("Added new peer to routing table: {:?}", peer) } KademliaEvent::OutboundQueryCompleted { id, result, .. } => { - info!("Query ID: {:?}", id); - if let libp2p::kad::QueryResult::Bootstrap(result) = result { - match result { + match &result { + QueryResult::GetClosestPeers(result) => match result { + Ok(res) => { + info!("GetClosestPeers finished successfully: {:?}", res.peers); + } + Err(e) => info!("{:?}", e), + }, + QueryResult::Bootstrap(result) => match result { Ok(res) => { info!("Bootstrapping finished successfully: {:?}", res.peer) } Err(e) => info!("{:?}", e), - } - } - } - KademliaEvent::RoutablePeer { peer, address } => { - self.swarm - .behaviour_mut() - .kademlia - .add_address(&peer, address); + }, + _ => {} + }; + // Send query results back so that we can use that information. + self.query_result.insert(id, result); } _ => {} }, From c53f9a12f5bf14340fc774885de31cdda7ebffe1 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 05:01:46 +0530 Subject: [PATCH 38/52] Remove unused imports. --- Cargo.lock | 12 ++++++++++++ crates/spartan-farmer/src/commands/dht/client.rs | 1 - crates/spartan-farmer/src/commands/dht/core.rs | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b484e19492..246fcdc62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7789,9 +7789,21 @@ dependencies = [ "once_cell", "pin-project-lite 0.2.7", "signal-hook-registry", + "tokio-macros", "winapi 0.3.9", ] +[[package]] +name = "tokio-macros" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.22.0" diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 2e87b848a8..802cfef45f 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,6 +1,5 @@ use super::*; use super::{core::create_node, eventloop::EventLoop}; -use libp2p::multiaddr::Protocol; #[derive(Copy, Clone, Debug)] pub enum ClientType { diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index a0046df391..6230abb537 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -64,6 +64,6 @@ fn dial_bootstrap(swarm: &mut Swarm, nodes: &Vec) { let addr = Multiaddr::from_str(parts[0]).unwrap(); let peer = PeerId::from_str(parts[1]).unwrap(); swarm.behaviour_mut().kademlia.add_address(&peer, addr); - // swarm.dial_addr(Multiaddr::from_str(node).unwrap()).unwrap(); + swarm.dial_addr(Multiaddr::from_str(node).unwrap()).unwrap(); } } From c11574b254637281dd5a15b0d1963988ccd83719 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 05:02:08 +0530 Subject: [PATCH 39/52] Add test for `bootstrap`. --- .../spartan-farmer/src/commands/dht/test.rs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/commands/dht/test.rs index 4563e55b7b..b5cd9640bb 100644 --- a/crates/spartan-farmer/src/commands/dht/test.rs +++ b/crates/spartan-farmer/src/commands/dht/test.rs @@ -1 +1,65 @@ use super::*; +use client as dht; +use client::{ClientConfig, ClientType}; + +#[tokio::test] +async fn bootstrap_working() { + // NOTE: There is no difference between a bootstrap and normal node. Under the hood, they both + // are the same things. + + let mut clients = Vec::new(); + let mut peeraddr = Vec::new(); + + // Consider 5 peers: A, B, C, D and E. + while clients.len() < 5 { + let config = ClientConfig { + bootstrap_nodes: Default::default(), + client_type: ClientType::Normal, + listen_addr: None, + }; + + let (mut client, eventloop) = dht::create_connection(&config); + + tokio::spawn(async move { eventloop.run().await }); + + client + .start_listening("/ip4/0.0.0.0/tcp/0".parse().unwrap()) + .await; + + let peerid = client.peerid.clone(); + let listen_addr = client.listeners().await; + + if listen_addr.is_empty() { + continue; + } + + clients.push(client); + peeraddr.push((peerid, listen_addr[0].to_owned())); + } + + // Connect A --> B, B --> C. + for i in 1..3 { + let peerid = peeraddr[i - 1].0.clone(); + let addr = peeraddr[i - 1].1.clone(); + clients[i].dial(peerid, addr).await; + println!("{}", i); + } + + // Connect D --> E. + for i in 3..5 { + let peerid = peeraddr[i - 1].0.clone(); + let addr = peeraddr[i - 1].1.clone(); + clients[i].dial(peerid, addr).await; + } + + // Connect A --> D. + let peerid = peeraddr[3].0.clone(); + let addr = peeraddr[3].1.clone(); + clients[0].dial(peerid, addr).await; + + clients[0].bootstrap().await; + + let known_peers = clients[0].known_peers().await; + + assert!(known_peers.contains(&peerid)); +} From dd2ab067d27731d873cbb9a404a753b2e7ba4786 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 05:04:32 +0530 Subject: [PATCH 40/52] Remove commented code --- crates/spartan-farmer/src/commands/dht/client.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 802cfef45f..538b886452 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -179,16 +179,8 @@ impl Client { if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { sender.send(qid).unwrap(); } - // match eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - // Ok(_qid) => sender.send(Ok(())).unwrap(), - // Err(e) => sender.send(Err(Box::new(e))).unwrap(), - // } } ClientEvent::RandomWalk { sender, key } => { - // NOTE: An interesting fact, that I have noticed is that Kademlia is not - // bidirectional. For example, if Peer 1 adds Peer 2 to its routing table, Peer 2 - // will not add Peer 1 to its routing table. - let key = match key { Some(peerid) => peerid, None => PeerId::random(), @@ -220,11 +212,6 @@ impl Client { .kademlia .add_address(&peer, addr.clone()); - // eventloop - // .swarm - // .dial_addr(addr.with(Protocol::P2p(peer.into()))) - // .unwrap(); - sender.send(Ok(())).unwrap(); } ClientEvent::Listeners { sender } => { From 7e271cdc1b8f2dd1bc802cce8ebd46d9adb45e78 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 05:54:51 +0530 Subject: [PATCH 41/52] Corrections to Bootstrap test --- .../spartan-farmer/src/commands/dht/client.rs | 3 +++ crates/spartan-farmer/src/commands/dht/test.rs | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 538b886452..5a14edf9d7 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,5 +1,6 @@ use super::*; use super::{core::create_node, eventloop::EventLoop}; +use libp2p::multiaddr::Protocol; #[derive(Copy, Clone, Debug)] pub enum ClientType { @@ -212,6 +213,8 @@ impl Client { .kademlia .add_address(&peer, addr.clone()); + eventloop.swarm.dial_addr(addr).unwrap(); + sender.send(Ok(())).unwrap(); } ClientEvent::Listeners { sender } => { diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/commands/dht/test.rs index b5cd9640bb..336753c907 100644 --- a/crates/spartan-farmer/src/commands/dht/test.rs +++ b/crates/spartan-farmer/src/commands/dht/test.rs @@ -39,17 +39,16 @@ async fn bootstrap_working() { // Connect A --> B, B --> C. for i in 1..3 { - let peerid = peeraddr[i - 1].0.clone(); - let addr = peeraddr[i - 1].1.clone(); - clients[i].dial(peerid, addr).await; - println!("{}", i); + let peerid = peeraddr[i].0.clone(); + let addr = peeraddr[i].1.clone(); + clients[i - 1].dial(peerid, addr).await; } // Connect D --> E. - for i in 3..5 { - let peerid = peeraddr[i - 1].0.clone(); - let addr = peeraddr[i - 1].1.clone(); - clients[i].dial(peerid, addr).await; + for i in 4..5 { + let peerid = peeraddr[i].0.clone(); + let addr = peeraddr[i].1.clone(); + clients[i - 1].dial(peerid, addr).await; } // Connect A --> D. @@ -57,9 +56,11 @@ async fn bootstrap_working() { let addr = peeraddr[3].1.clone(); clients[0].dial(peerid, addr).await; + // A should find E. clients[0].bootstrap().await; let known_peers = clients[0].known_peers().await; + let peerid = peeraddr[4].0.clone(); assert!(known_peers.contains(&peerid)); } From 6336ff12e96abfb44746df27d58f5b3cb4e1b79f Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 15:22:24 +0530 Subject: [PATCH 42/52] Add `QueryResult` event. --- Cargo.lock | 1 + crates/spartan-farmer/Cargo.toml | 2 +- .../spartan-farmer/src/commands/dht/client.rs | 66 ++++++++++++++++--- .../src/commands/dht/eventloop.rs | 4 +- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 246fcdc62e..cbaafde50a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7787,6 +7787,7 @@ dependencies = [ "mio 0.7.13", "num_cpus", "once_cell", + "parking_lot 0.11.2", "pin-project-lite 0.2.7", "signal-hook-registry", "tokio-macros", diff --git a/crates/spartan-farmer/Cargo.toml b/crates/spartan-farmer/Cargo.toml index 7e3ea980f6..cc045fcc82 100644 --- a/crates/spartan-farmer/Cargo.toml +++ b/crates/spartan-farmer/Cargo.toml @@ -30,7 +30,7 @@ serde_json = "1.0.64" schnorrkel = "0.10.1" spartan-codec = "0.1.0" thiserror = "1.0.24" -tokio = { version = "1.11.0", features = [ "rt", "macros", "fs" ] } +tokio = { version = "1.11.0", features = [ "full" ] } [dependencies.jsonrpsee] features = ["client"] diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 5a14edf9d7..0fd46f0b11 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -1,6 +1,6 @@ use super::*; use super::{core::create_node, eventloop::EventLoop}; -use libp2p::multiaddr::Protocol; +use libp2p::kad::QueryInfo; #[derive(Copy, Clone, Debug)] pub enum ClientType { @@ -48,7 +48,7 @@ pub enum ClientEvent { // Read Kademlia Query Result. QueryResult { qid: QueryId, - sender: oneshot::Sender, + sender: oneshot::Sender, }, } @@ -70,7 +70,7 @@ impl Client { } // Read the Query Result for a specific Kademlia query. - async fn query_result(&mut self, qid: QueryId) -> QueryResult { + pub async fn query_result(&mut self, qid: QueryId) -> String { let (sender, recv) = oneshot::channel(); self.client_tx @@ -145,7 +145,7 @@ impl Client { } // Sync with other peers on the DHT. (GetClosestPeer) - pub async fn random_walk(&mut self, key: Option) { + pub async fn random_walk(&mut self, key: Option) -> QueryId { let (sender, recv) = oneshot::channel(); self.client_tx @@ -154,11 +154,11 @@ impl Client { .unwrap(); // Check if the Bootstrap was processed, properly. - let _ = recv.await.expect("Failed to Random Walk."); + recv.await.expect("Failed to Random Walk.") } // Bootstrap - pub async fn bootstrap(&mut self) { + pub async fn bootstrap(&mut self) -> QueryId { let (sender, recv) = oneshot::channel(); self.client_tx @@ -167,7 +167,7 @@ impl Client { .unwrap(); // Check if the Bootstrap was processed, properly. - let _qid = recv.await.expect("Failed to bootstrap."); + recv.await.expect("Failed to bootstrap.") } pub fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { @@ -229,9 +229,55 @@ impl Client { .unwrap(); } ClientEvent::QueryResult { qid, sender } => { - sender - .send(eventloop.query_result.remove(&qid).unwrap()) - .unwrap(); + let mut result = String::new(); + + if eventloop.query_result.contains_key(&qid) { + result = match eventloop.query_result.remove(&qid).unwrap() { + QueryResult::Bootstrap(result) => match result { + Ok(result) => format!( + "[RESULT] This query still has {:?} peers remaining.", + result.num_remaining + ), + Err(e) => format!("{:?}", e), + }, + QueryResult::GetClosestPeers(result) => match result { + Ok(result) => { + format!("This query produced {:?} peers.", result.peers.len()) + } + Err(e) => format!("{:?}", e), + }, + _ => "Unknown QueryResult Type".to_string(), + }; + } else { + let query = eventloop + .swarm + .behaviour_mut() + .kademlia + .query(&qid) + .unwrap(); + + let stats = format!( + "Total Requests: {}\nFailed: {}\nSucceded: {}\nPending: {}\n", + query.stats().num_requests(), + query.stats().num_failures(), + query.stats().num_successes(), + query.stats().num_pending() + ); + + let info = match query.info() { + QueryInfo::Bootstrap { remaining, .. } => { + format!( + "[INFO] This query still has {:?} peers remaining.", + remaining + ) + } + _ => "Unknown QueryInfo Type".to_string(), + }; + + result = stats.clone() + &info; + } + + sender.send(result).unwrap(); } } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index c2fe4430d2..bb683f52bc 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -47,13 +47,13 @@ impl EventLoop { } KademliaEvent::OutboundQueryCompleted { id, result, .. } => { match &result { - QueryResult::GetClosestPeers(result) => match result { + QueryResult::GetClosestPeers(rnd_walk_result) => match rnd_walk_result { Ok(res) => { info!("GetClosestPeers finished successfully: {:?}", res.peers); } Err(e) => info!("{:?}", e), }, - QueryResult::Bootstrap(result) => match result { + QueryResult::Bootstrap(bootstrap_result) => match bootstrap_result { Ok(res) => { info!("Bootstrapping finished successfully: {:?}", res.peer) } From 8f3d6dd8a453d3357dd6ab0360fe6c2dda96ddc7 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 15:23:05 +0530 Subject: [PATCH 43/52] Inelegant fix for BootStrap test. --- crates/spartan-farmer/src/commands/dht/test.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/commands/dht/test.rs index 336753c907..fccd4c0ea1 100644 --- a/crates/spartan-farmer/src/commands/dht/test.rs +++ b/crates/spartan-farmer/src/commands/dht/test.rs @@ -20,7 +20,7 @@ async fn bootstrap_working() { let (mut client, eventloop) = dht::create_connection(&config); - tokio::spawn(async move { eventloop.run().await }); + tokio::spawn(eventloop.run()); client .start_listening("/ip4/0.0.0.0/tcp/0".parse().unwrap()) @@ -57,7 +57,17 @@ async fn bootstrap_working() { clients[0].dial(peerid, addr).await; // A should find E. - clients[0].bootstrap().await; + let qid = clients[0].bootstrap().await; + + let mut result = String::default(); + // Keep qeurying the result until we get the event we are looking for. + loop { + result = clients[0].query_result(qid).await; + if result.contains("[RESULT] This query still has 0 peers remaining.") { + break; + } + } + println!("{:?}", result); let known_peers = clients[0].known_peers().await; let peerid = peeraddr[4].0.clone(); From 4c64bd668f48b1c36617eb9ef77a4d7fb4ec3b4e Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Wed, 22 Sep 2021 18:39:29 +0530 Subject: [PATCH 44/52] Code ready for review (unless, clippy gets in the way) --- Cargo.lock | 1 - crates/spartan-farmer/Cargo.toml | 2 +- .../spartan-farmer/src/commands/dht/client.rs | 21 +++++++++++++------ .../src/commands/dht/eventloop.rs | 9 ++++---- .../spartan-farmer/src/commands/dht/test.rs | 4 +--- crates/spartan-farmer/src/commands/farm.rs | 3 --- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcff9c1295..951e4a06aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7819,7 +7819,6 @@ dependencies = [ "mio 0.7.13", "num_cpus", "once_cell", - "parking_lot 0.11.2", "pin-project-lite 0.2.7", "signal-hook-registry", "tokio-macros", diff --git a/crates/spartan-farmer/Cargo.toml b/crates/spartan-farmer/Cargo.toml index 1e88809bfb..01c0cdf5f0 100644 --- a/crates/spartan-farmer/Cargo.toml +++ b/crates/spartan-farmer/Cargo.toml @@ -30,7 +30,7 @@ serde_json = "1.0.64" schnorrkel = "0.10.1" spartan-codec = "0.1.0" thiserror = "1.0.24" -tokio = { version = "1.11.0", features = [ "full" ] } +tokio = { version = "1.11.0", features = [ "macros" ] } [dependencies.jsonrpsee] features = ["client"] diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 0fd46f0b11..36cc4267d3 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -21,15 +21,18 @@ pub enum ClientEvent { sender: oneshot::Sender, }, // Kademlia Random Walk for Peer Discovery. (GetClosestPeer) + #[allow(dead_code)] RandomWalk { key: Option, sender: oneshot::Sender, }, // List all known peers. + #[allow(dead_code)] KnownPeers { sender: oneshot::Sender>, }, // Dial another peer. + #[allow(dead_code)] Dial { addr: Multiaddr, peer: PeerId, @@ -42,10 +45,12 @@ pub enum ClientEvent { sender: oneshot::Sender, }, // Get all listening addresses. + #[allow(dead_code)] Listeners { sender: oneshot::Sender>, }, // Read Kademlia Query Result. + #[allow(dead_code)] QueryResult { qid: QueryId, sender: oneshot::Sender, @@ -70,6 +75,8 @@ impl Client { } // Read the Query Result for a specific Kademlia query. + // This method returns information about pending as well as finsihed queries. + #[allow(dead_code)] pub async fn query_result(&mut self, qid: QueryId) -> String { let (sender, recv) = oneshot::channel(); @@ -86,6 +93,7 @@ impl Client { } // Get the list of all addresses we are listening on. + #[allow(dead_code)] pub async fn listeners(&mut self) -> Vec { let (sender, recv) = oneshot::channel(); @@ -102,6 +110,7 @@ impl Client { } // Dial another node using Peer Id and Address. + #[allow(dead_code)] pub async fn dial(&mut self, peer: PeerId, addr: Multiaddr) { let (sender, recv) = oneshot::channel(); @@ -114,6 +123,7 @@ impl Client { } // Returns the list of all the peers the client has in its Routing table. + #[allow(dead_code)] pub async fn known_peers(&mut self) -> Vec { let (sender, recv) = oneshot::channel(); @@ -145,6 +155,7 @@ impl Client { } // Sync with other peers on the DHT. (GetClosestPeer) + #[allow(dead_code)] pub async fn random_walk(&mut self, key: Option) -> QueryId { let (sender, recv) = oneshot::channel(); @@ -229,10 +240,8 @@ impl Client { .unwrap(); } ClientEvent::QueryResult { qid, sender } => { - let mut result = String::new(); - if eventloop.query_result.contains_key(&qid) { - result = match eventloop.query_result.remove(&qid).unwrap() { + let result = match eventloop.query_result.remove(&qid).unwrap() { QueryResult::Bootstrap(result) => match result { Ok(result) => format!( "[RESULT] This query still has {:?} peers remaining.", @@ -248,6 +257,8 @@ impl Client { }, _ => "Unknown QueryResult Type".to_string(), }; + + sender.send(result).unwrap(); } else { let query = eventloop .swarm @@ -274,10 +285,8 @@ impl Client { _ => "Unknown QueryInfo Type".to_string(), }; - result = stats.clone() + &info; + sender.send(stats.clone() + &info).unwrap(); } - - sender.send(result).unwrap(); } } } diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/commands/dht/eventloop.rs index bb683f52bc..6fa859b248 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/commands/dht/eventloop.rs @@ -49,14 +49,15 @@ impl EventLoop { match &result { QueryResult::GetClosestPeers(rnd_walk_result) => match rnd_walk_result { Ok(res) => { - info!("GetClosestPeers finished successfully: {:?}", res.peers); + info!( + "Kademlia Random Walk finished successfully: {:?}", + res.peers + ); } Err(e) => info!("{:?}", e), }, QueryResult::Bootstrap(bootstrap_result) => match bootstrap_result { - Ok(res) => { - info!("Bootstrapping finished successfully: {:?}", res.peer) - } + Ok(_res) => info!("Bootstrapping finished successfully."), Err(e) => info!("{:?}", e), }, _ => {} diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/commands/dht/test.rs index fccd4c0ea1..1bc35c867e 100644 --- a/crates/spartan-farmer/src/commands/dht/test.rs +++ b/crates/spartan-farmer/src/commands/dht/test.rs @@ -59,15 +59,13 @@ async fn bootstrap_working() { // A should find E. let qid = clients[0].bootstrap().await; - let mut result = String::default(); // Keep qeurying the result until we get the event we are looking for. loop { - result = clients[0].query_result(qid).await; + let result = clients[0].query_result(qid).await; if result.contains("[RESULT] This query still has 0 peers remaining.") { break; } } - println!("{:?}", result); let known_peers = clients[0].known_peers().await; let peerid = peeraddr[4].0.clone(); diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 07862e1a33..4f306d7df5 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -113,9 +113,6 @@ pub(crate) async fn farm( .await; }; - // TODO: Ideally, this should be performed periodically after a fixed time interval. - // Peer Discovery needs to be a repetitive process, to make sure no peers carry stale - // information. dht_client.bootstrap().await; info!("Opening plot"); From a38dd71233e45ae224cf806d7965e67c7aee6b88 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Thu, 23 Sep 2021 14:06:32 +0530 Subject: [PATCH 45/52] Appease clippy. --- crates/spartan-farmer/src/commands/dht/client.rs | 12 +++--------- crates/spartan-farmer/src/commands/dht/core.rs | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/commands/dht/client.rs index 36cc4267d3..a429f1d89b 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/commands/dht/client.rs @@ -211,7 +211,7 @@ impl Client { for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { for record in bucket.iter() { - result.push(record.node.key.preimage().clone()); + result.push(*record.node.key.preimage()); } } @@ -230,13 +230,7 @@ impl Client { } ClientEvent::Listeners { sender } => { sender - .send( - eventloop - .swarm - .listeners() - .map(|addr| addr.clone()) - .collect(), - ) + .send(eventloop.swarm.listeners().cloned().collect::>()) .unwrap(); } ClientEvent::QueryResult { qid, sender } => { @@ -285,7 +279,7 @@ impl Client { _ => "Unknown QueryInfo Type".to_string(), }; - sender.send(stats.clone() + &info).unwrap(); + sender.send(stats + &info).unwrap(); } } } diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/commands/dht/core.rs index 6230abb537..24da47464e 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/commands/dht/core.rs @@ -58,12 +58,12 @@ fn create_swarm(peerid: PeerId, key: identity::Keypair) -> Swarm, nodes: &Vec) { +fn dial_bootstrap(swarm: &mut Swarm, nodes: &[String]) { for node in nodes { let parts: Vec<&str> = node.split("/p2p/").collect(); let addr = Multiaddr::from_str(parts[0]).unwrap(); let peer = PeerId::from_str(parts[1]).unwrap(); swarm.behaviour_mut().kademlia.add_address(&peer, addr); - swarm.dial_addr(Multiaddr::from_str(node).unwrap()).unwrap(); + // swarm.dial_addr(Multiaddr::from_str(node).unwrap()).unwrap(); } } From 4e3e476fbcfe5078cb966cc6548b5ed809fda743 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Thu, 23 Sep 2021 19:11:18 +0530 Subject: [PATCH 46/52] Add rustfmt.toml and rewrite dependencies --- .gitignore | 1 - crates/spartan-farmer/Cargo.toml | 16 ++++++++++------ rustfmt.toml | 4 ++++ 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 rustfmt.toml diff --git a/.gitignore b/.gitignore index 778e1617eb..e53981c959 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .gitignore /.idea /target -**/rustfmt.toml .vscode/ diff --git a/crates/spartan-farmer/Cargo.toml b/crates/spartan-farmer/Cargo.toml index 01c0cdf5f0..8d6d14f141 100644 --- a/crates/spartan-farmer/Cargo.toml +++ b/crates/spartan-farmer/Cargo.toml @@ -30,12 +30,17 @@ serde_json = "1.0.64" schnorrkel = "0.10.1" spartan-codec = "0.1.0" thiserror = "1.0.24" -tokio = { version = "1.11.0", features = [ "macros" ] } +tokio = "1.11.0" [dependencies.jsonrpsee] features = ["client"] version = "0.2.0" +[dependencies.libp2p] +default-features = false +features = [ "noise", "mplex", "kad", "tcp-tokio" ] +version = "0.39.1" + [dependencies.rocksdb] # This disables compression algorithms that cause issues during linking due to # https://github.com/rust-rocksdb/rust-rocksdb/issues/514 @@ -48,11 +53,6 @@ version = "0.17.0" features = ["derive"] version = "1.0.125" -[dependencies.libp2p] -version = "0.39.1" -features = [ "noise", "mplex", "kad", "tcp-tokio" ] -default-features = false - [dependencies.subspace-core-primitives] version = "0.1.0" path = "../subspace-core-primitives" @@ -63,3 +63,7 @@ rand = "0.8.4" [dev-dependencies.async-std] features = ["attributes"] version = "1.9.0" + +[dev-dependencies.tokio] +features = [ "macros" ] +version = "1.11.0" diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000..20ad52fa9b --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,4 @@ +ignore = [ + # This is where Substrate sources with custom formatting are, don't touch them + "/substrate", +] From 77496fe7a8dd1a2dea56fdaa6c78c9bab7050fbc Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Thu, 23 Sep 2021 19:31:24 +0530 Subject: [PATCH 47/52] Move `dht` module out of `commands` and seperate imports. --- crates/spartan-farmer/src/commands/dht.rs | 40 ------------------- crates/spartan-farmer/src/dht.rs | 12 ++++++ .../src/{commands => }/dht/client.rs | 17 +++++++- .../src/{commands => }/dht/core.rs | 16 +++++++- .../src/{commands => }/dht/eventloop.rs | 15 ++++++- .../src/{commands => }/dht/test.rs | 0 crates/spartan-farmer/src/main.rs | 1 + 7 files changed, 58 insertions(+), 43 deletions(-) delete mode 100644 crates/spartan-farmer/src/commands/dht.rs create mode 100644 crates/spartan-farmer/src/dht.rs rename crates/spartan-farmer/src/{commands => }/dht/client.rs (96%) rename crates/spartan-farmer/src/{commands => }/dht/core.rs (82%) rename crates/spartan-farmer/src/{commands => }/dht/eventloop.rs (88%) rename crates/spartan-farmer/src/{commands => }/dht/test.rs (100%) diff --git a/crates/spartan-farmer/src/commands/dht.rs b/crates/spartan-farmer/src/commands/dht.rs deleted file mode 100644 index 0fd0825271..0000000000 --- a/crates/spartan-farmer/src/commands/dht.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Stuff for Kademlia -use libp2p::kad::{record::store::MemoryStore, Kademlia, KademliaEvent, QueryId, QueryResult}; - -// Stuff for defining composed behaviour -use libp2p::NetworkBehaviour; - -// Stuff needed to create the swarm -use libp2p::core::{upgrade, Transport}; -use libp2p::identity; -use libp2p::mplex; -use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; -use libp2p::swarm::{SwarmBuilder, SwarmEvent}; -use libp2p::tcp::TokioTcpConfig; -use libp2p::{Multiaddr, PeerId, Swarm}; - -// Stuff needed to set up channels between Client API task and EventLoop task. -use futures::channel::{ - mpsc::{channel, Receiver, Sender}, - oneshot, -}; -use futures::prelude::*; -use futures::StreamExt; - -use log::info; - -type OneshotError = Box; -type OneshotType = Result<(), OneshotError>; - -// The Client API which the end-user is supposed to interact with. -pub mod client; -// Core libp2p activities like defining network behaviour and events, bootstrap-ing, -// creating of swarm and such... -mod core; -// EventLoop which actually processes libp2p SwarmEvents. The Client API interacts with the -// EventLoop to transfer and receieve data. -mod eventloop; - -// DHT related tests. -#[cfg(test)] -mod test; diff --git a/crates/spartan-farmer/src/dht.rs b/crates/spartan-farmer/src/dht.rs new file mode 100644 index 0000000000..e14bd07f7f --- /dev/null +++ b/crates/spartan-farmer/src/dht.rs @@ -0,0 +1,12 @@ +// The Client API which the end-user is supposed to interact with. +pub(crate) mod client; +// Core libp2p activities like defining network behaviour and events, bootstrap-ing, +// creating of swarm and such... +mod core; +// EventLoop which actually processes libp2p SwarmEvents. The Client API interacts with the +// EventLoop to transfer and receieve data. +mod eventloop; + +// DHT related tests. +#[cfg(test)] +mod test; diff --git a/crates/spartan-farmer/src/commands/dht/client.rs b/crates/spartan-farmer/src/dht/client.rs similarity index 96% rename from crates/spartan-farmer/src/commands/dht/client.rs rename to crates/spartan-farmer/src/dht/client.rs index a429f1d89b..4ec771dd1b 100644 --- a/crates/spartan-farmer/src/commands/dht/client.rs +++ b/crates/spartan-farmer/src/dht/client.rs @@ -1,4 +1,19 @@ -use super::*; +// Stuff for Kademlia +use libp2p::kad::{QueryId, QueryResult}; + +// Stuff needed to create the swarm +use libp2p::{Multiaddr, PeerId}; + +// Stuff needed to set up channels between Client API task and EventLoop task. +use futures::channel::{ + mpsc::{channel, Sender}, + oneshot, +}; +use futures::prelude::*; + +type OneshotError = Box; +type OneshotType = Result<(), OneshotError>; + use super::{core::create_node, eventloop::EventLoop}; use libp2p::kad::QueryInfo; diff --git a/crates/spartan-farmer/src/commands/dht/core.rs b/crates/spartan-farmer/src/dht/core.rs similarity index 82% rename from crates/spartan-farmer/src/commands/dht/core.rs rename to crates/spartan-farmer/src/dht/core.rs index 24da47464e..5df808854e 100644 --- a/crates/spartan-farmer/src/commands/dht/core.rs +++ b/crates/spartan-farmer/src/dht/core.rs @@ -1,6 +1,20 @@ +// Stuff for Kademlia +use libp2p::kad::{record::store::MemoryStore, Kademlia, KademliaEvent}; + +// Stuff for defining composed behaviour +use libp2p::NetworkBehaviour; + +// Stuff needed to create the swarm +use libp2p::core::{upgrade, Transport}; +use libp2p::identity; +use libp2p::mplex; +use libp2p::noise::{Keypair, NoiseConfig, X25519Spec}; +use libp2p::swarm::SwarmBuilder; +use libp2p::tcp::TokioTcpConfig; +use libp2p::{Multiaddr, PeerId, Swarm}; + // Pull imports from the parent module use super::client::ClientConfig; -use super::*; use std::str::FromStr; #[derive(NetworkBehaviour)] diff --git a/crates/spartan-farmer/src/commands/dht/eventloop.rs b/crates/spartan-farmer/src/dht/eventloop.rs similarity index 88% rename from crates/spartan-farmer/src/commands/dht/eventloop.rs rename to crates/spartan-farmer/src/dht/eventloop.rs index 6fa859b248..470ba63d53 100644 --- a/crates/spartan-farmer/src/commands/dht/eventloop.rs +++ b/crates/spartan-farmer/src/dht/eventloop.rs @@ -1,4 +1,17 @@ -use super::*; +// Stuff for Kademlia +use libp2p::kad::{KademliaEvent, QueryId, QueryResult}; + +use libp2p::{swarm::SwarmEvent, Swarm}; + +// Stuff needed to set up channels between Client API task and EventLoop task. +use futures::channel::mpsc::Receiver; +use futures::StreamExt; + +use log::info; + +type OneshotError = Box; +type OneshotType = Result<(), OneshotError>; + use super::{ client::{Client, ClientEvent}, core::{ComposedBehaviour, ComposedEvent}, diff --git a/crates/spartan-farmer/src/commands/dht/test.rs b/crates/spartan-farmer/src/dht/test.rs similarity index 100% rename from crates/spartan-farmer/src/commands/dht/test.rs rename to crates/spartan-farmer/src/dht/test.rs diff --git a/crates/spartan-farmer/src/main.rs b/crates/spartan-farmer/src/main.rs index 5f436d0278..0340309d93 100644 --- a/crates/spartan-farmer/src/main.rs +++ b/crates/spartan-farmer/src/main.rs @@ -17,6 +17,7 @@ mod commands; mod crypto; +mod dht; mod plot; mod utils; From 2408b66e622d9fcbe26c412636bf52b2044ecb54 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Thu, 23 Sep 2021 20:18:54 +0530 Subject: [PATCH 48/52] Reassign scopes to objects in dht module --- crates/spartan-farmer/src/commands.rs | 1 - crates/spartan-farmer/src/commands/farm.rs | 7 +- crates/spartan-farmer/src/dht.rs | 4 +- crates/spartan-farmer/src/dht/client.rs | 185 ++++++++++----------- crates/spartan-farmer/src/dht/core.rs | 6 +- crates/spartan-farmer/src/dht/eventloop.rs | 13 +- 6 files changed, 103 insertions(+), 113 deletions(-) diff --git a/crates/spartan-farmer/src/commands.rs b/crates/spartan-farmer/src/commands.rs index 7174e9cede..d700585e86 100644 --- a/crates/spartan-farmer/src/commands.rs +++ b/crates/spartan-farmer/src/commands.rs @@ -1,4 +1,3 @@ -mod dht; mod farm; mod plot; diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index 4f306d7df5..fce4c4d917 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -1,7 +1,4 @@ -use super::dht::{ - client as dht, - client::{ClientConfig, ClientType}, -}; +use crate::dht::{create_connection, ClientConfig, ClientType}; use crate::plot::Plot; use crate::{crypto, Salt, Tag, PRIME_SIZE_BYTES, SIGNING_CONTEXT}; use async_std::task; @@ -102,7 +99,7 @@ pub(crate) async fn farm( }; info!("Connecting to DHT"); - let (mut dht_client, dht_eventloop) = dht::create_connection(&config); + let (mut dht_client, dht_eventloop) = create_connection(&config); tokio::spawn(async move { dht_eventloop.run().await }); info!("My Peer ID is: {:?}", dht_client.peerid); diff --git a/crates/spartan-farmer/src/dht.rs b/crates/spartan-farmer/src/dht.rs index e14bd07f7f..df216ad67a 100644 --- a/crates/spartan-farmer/src/dht.rs +++ b/crates/spartan-farmer/src/dht.rs @@ -1,5 +1,5 @@ // The Client API which the end-user is supposed to interact with. -pub(crate) mod client; +mod client; // Core libp2p activities like defining network behaviour and events, bootstrap-ing, // creating of swarm and such... mod core; @@ -10,3 +10,5 @@ mod eventloop; // DHT related tests. #[cfg(test)] mod test; + +pub(crate) use client::{create_connection, ClientConfig, ClientType}; diff --git a/crates/spartan-farmer/src/dht/client.rs b/crates/spartan-farmer/src/dht/client.rs index 4ec771dd1b..4ce9eac063 100644 --- a/crates/spartan-farmer/src/dht/client.rs +++ b/crates/spartan-farmer/src/dht/client.rs @@ -29,7 +29,7 @@ pub enum ClientType { Normal, } -pub enum ClientEvent { +pub(super) enum ClientEvent { // Event for adding a listening address. Listen { addr: Multiaddr, @@ -59,11 +59,6 @@ pub enum ClientEvent { Bootstrap { sender: oneshot::Sender, }, - // Get all listening addresses. - #[allow(dead_code)] - Listeners { - sender: oneshot::Sender>, - }, // Read Kademlia Query Result. #[allow(dead_code)] QueryResult { @@ -85,7 +80,7 @@ pub struct Client { } impl Client { - pub fn new(peerid: PeerId, client_tx: Sender) -> Self { + fn new(peerid: PeerId, client_tx: Sender) -> Self { Client { peerid, client_tx } } @@ -195,107 +190,107 @@ impl Client { // Check if the Bootstrap was processed, properly. recv.await.expect("Failed to bootstrap.") } +} + +pub(super) fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { + match event { + ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { + Ok(_) => sender.send(Ok(())).unwrap(), + Err(e) => sender.send(Err(Box::new(e))).unwrap(), + }, + ClientEvent::Bootstrap { sender } => { + if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { + sender.send(qid).unwrap(); + } + } + ClientEvent::RandomWalk { sender, key } => { + let key = match key { + Some(peerid) => peerid, + None => PeerId::random(), + }; + + let qid = eventloop + .swarm + .behaviour_mut() + .kademlia + .get_closest_peers(key); + + sender.send(qid).unwrap(); + } + ClientEvent::KnownPeers { sender } => { + let mut result = Vec::new(); - pub fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { - match event { - ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { - Ok(_) => sender.send(Ok(())).unwrap(), - Err(e) => sender.send(Err(Box::new(e))).unwrap(), - }, - ClientEvent::Bootstrap { sender } => { - if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - sender.send(qid).unwrap(); + for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { + for record in bucket.iter() { + result.push(*record.node.key.preimage()); } } - ClientEvent::RandomWalk { sender, key } => { - let key = match key { - Some(peerid) => peerid, - None => PeerId::random(), - }; - let qid = eventloop - .swarm - .behaviour_mut() - .kademlia - .get_closest_peers(key); + sender.send(result).unwrap(); + } + ClientEvent::Dial { addr, peer, sender } => { + eventloop + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer, addr.clone()); - sender.send(qid).unwrap(); - } - ClientEvent::KnownPeers { sender } => { - let mut result = Vec::new(); + eventloop.swarm.dial_addr(addr).unwrap(); - for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { - for record in bucket.iter() { - result.push(*record.node.key.preimage()); - } - } + sender.send(Ok(())).unwrap(); + } + ClientEvent::Listeners { sender } => { + sender + .send(eventloop.swarm.listeners().cloned().collect::>()) + .unwrap(); + } + ClientEvent::QueryResult { qid, sender } => { + if eventloop.query_result.contains_key(&qid) { + let result = match eventloop.query_result.remove(&qid).unwrap() { + QueryResult::Bootstrap(result) => match result { + Ok(result) => format!( + "[RESULT] This query still has {:?} peers remaining.", + result.num_remaining + ), + Err(e) => format!("{:?}", e), + }, + QueryResult::GetClosestPeers(result) => match result { + Ok(result) => { + format!("This query produced {:?} peers.", result.peers.len()) + } + Err(e) => format!("{:?}", e), + }, + _ => "Unknown QueryResult Type".to_string(), + }; sender.send(result).unwrap(); - } - ClientEvent::Dial { addr, peer, sender } => { - eventloop + } else { + let query = eventloop .swarm .behaviour_mut() .kademlia - .add_address(&peer, addr.clone()); - - eventloop.swarm.dial_addr(addr).unwrap(); - - sender.send(Ok(())).unwrap(); - } - ClientEvent::Listeners { sender } => { - sender - .send(eventloop.swarm.listeners().cloned().collect::>()) + .query(&qid) .unwrap(); - } - ClientEvent::QueryResult { qid, sender } => { - if eventloop.query_result.contains_key(&qid) { - let result = match eventloop.query_result.remove(&qid).unwrap() { - QueryResult::Bootstrap(result) => match result { - Ok(result) => format!( - "[RESULT] This query still has {:?} peers remaining.", - result.num_remaining - ), - Err(e) => format!("{:?}", e), - }, - QueryResult::GetClosestPeers(result) => match result { - Ok(result) => { - format!("This query produced {:?} peers.", result.peers.len()) - } - Err(e) => format!("{:?}", e), - }, - _ => "Unknown QueryResult Type".to_string(), - }; - - sender.send(result).unwrap(); - } else { - let query = eventloop - .swarm - .behaviour_mut() - .kademlia - .query(&qid) - .unwrap(); - - let stats = format!( - "Total Requests: {}\nFailed: {}\nSucceded: {}\nPending: {}\n", - query.stats().num_requests(), - query.stats().num_failures(), - query.stats().num_successes(), - query.stats().num_pending() - ); - - let info = match query.info() { - QueryInfo::Bootstrap { remaining, .. } => { - format!( - "[INFO] This query still has {:?} peers remaining.", - remaining - ) - } - _ => "Unknown QueryInfo Type".to_string(), - }; - sender.send(stats + &info).unwrap(); - } + let stats = format!( + "Total Requests: {}\nFailed: {}\nSucceded: {}\nPending: {}\n", + query.stats().num_requests(), + query.stats().num_failures(), + query.stats().num_successes(), + query.stats().num_pending() + ); + + let info = match query.info() { + QueryInfo::Bootstrap { remaining, .. } => { + format!( + "[INFO] This query still has {:?} peers remaining.", + remaining + ) + } + _ => "Unknown QueryInfo Type".to_string(), + }; + + sender.send(stats + &info).unwrap(); } } } diff --git a/crates/spartan-farmer/src/dht/core.rs b/crates/spartan-farmer/src/dht/core.rs index 5df808854e..959a13b30f 100644 --- a/crates/spartan-farmer/src/dht/core.rs +++ b/crates/spartan-farmer/src/dht/core.rs @@ -19,11 +19,11 @@ use std::str::FromStr; #[derive(NetworkBehaviour)] #[behaviour(event_process = false, out_event = "ComposedEvent")] -pub struct ComposedBehaviour { +pub(super) struct ComposedBehaviour { pub kademlia: Kademlia, } -pub enum ComposedEvent { +pub(super) enum ComposedEvent { Kademlia(KademliaEvent), } @@ -33,7 +33,7 @@ impl From for ComposedEvent { } } -pub fn create_node(config: &ClientConfig) -> (PeerId, Swarm) { +pub(super) fn create_node(config: &ClientConfig) -> (PeerId, Swarm) { // Generate IDs. let key = identity::Keypair::generate_ed25519(); let peerid = PeerId::from_public_key(key.public()); diff --git a/crates/spartan-farmer/src/dht/eventloop.rs b/crates/spartan-farmer/src/dht/eventloop.rs index 470ba63d53..6e62ccb557 100644 --- a/crates/spartan-farmer/src/dht/eventloop.rs +++ b/crates/spartan-farmer/src/dht/eventloop.rs @@ -9,26 +9,23 @@ use futures::StreamExt; use log::info; -type OneshotError = Box; -type OneshotType = Result<(), OneshotError>; - use super::{ - client::{Client, ClientEvent}, + client::{handle_client_event, ClientEvent}, core::{ComposedBehaviour, ComposedEvent}, }; use std::collections::HashMap; pub struct EventLoop { - pub swarm: Swarm, + pub(super) swarm: Swarm, // Channel to receive events from Client. client_rx: Receiver, // HashMap to send back QueryResults. - pub query_result: HashMap, + pub(super) query_result: HashMap, } impl EventLoop { // Create new event loop - pub fn new(swarm: Swarm, client_rx: Receiver) -> Self { + pub(super) fn new(swarm: Swarm, client_rx: Receiver) -> Self { EventLoop { swarm, client_rx, @@ -41,7 +38,7 @@ impl EventLoop { loop { futures::select! { client_event = self.client_rx.next() => if let Some(event) = client_event { - Client::handle_client_event(&mut self, event) + handle_client_event(&mut self, event) }, network_event = self.swarm.next() => match network_event { Some(event) => self.handle_network_event(event).await, From 8e638a38032693d0aed53ce9e8de36b056a9ddc0 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Thu, 23 Sep 2021 22:27:52 +0530 Subject: [PATCH 49/52] Remove ClientEvents that we are not using. --- crates/spartan-farmer/src/dht/client.rs | 111 +++++++++------------ crates/spartan-farmer/src/dht/eventloop.rs | 9 -- 2 files changed, 45 insertions(+), 75 deletions(-) diff --git a/crates/spartan-farmer/src/dht/client.rs b/crates/spartan-farmer/src/dht/client.rs index 4ce9eac063..5d9f91e1c5 100644 --- a/crates/spartan-farmer/src/dht/client.rs +++ b/crates/spartan-farmer/src/dht/client.rs @@ -29,44 +29,6 @@ pub enum ClientType { Normal, } -pub(super) enum ClientEvent { - // Event for adding a listening address. - Listen { - addr: Multiaddr, - sender: oneshot::Sender, - }, - // Kademlia Random Walk for Peer Discovery. (GetClosestPeer) - #[allow(dead_code)] - RandomWalk { - key: Option, - sender: oneshot::Sender, - }, - // List all known peers. - #[allow(dead_code)] - KnownPeers { - sender: oneshot::Sender>, - }, - // Dial another peer. - #[allow(dead_code)] - Dial { - addr: Multiaddr, - peer: PeerId, - sender: oneshot::Sender, - }, - // Bootstrap during the initial connection to the DHT. - // NOTE: All the bootstrap nodes must already be connected to the swarm before we can start the - // bootstrap process. - Bootstrap { - sender: oneshot::Sender, - }, - // Read Kademlia Query Result. - #[allow(dead_code)] - QueryResult { - qid: QueryId, - sender: oneshot::Sender, - }, -} - pub struct ClientConfig { pub bootstrap_nodes: Vec, // Vec<(Multiaddr, PeerId)>, pub client_type: ClientType, @@ -86,6 +48,7 @@ impl Client { // Read the Query Result for a specific Kademlia query. // This method returns information about pending as well as finsihed queries. + // TODO: We are using for testing. #[allow(dead_code)] pub async fn query_result(&mut self, qid: QueryId) -> String { let (sender, recv) = oneshot::channel(); @@ -103,6 +66,7 @@ impl Client { } // Get the list of all addresses we are listening on. + // TODO: We are using for testing. #[allow(dead_code)] pub async fn listeners(&mut self) -> Vec { let (sender, recv) = oneshot::channel(); @@ -120,6 +84,7 @@ impl Client { } // Dial another node using Peer Id and Address. + // TODO: We are using for testing. #[allow(dead_code)] pub async fn dial(&mut self, peer: PeerId, addr: Multiaddr) { let (sender, recv) = oneshot::channel(); @@ -133,6 +98,7 @@ impl Client { } // Returns the list of all the peers the client has in its Routing table. + // TODO: We are using for testing. #[allow(dead_code)] pub async fn known_peers(&mut self) -> Vec { let (sender, recv) = oneshot::channel(); @@ -164,20 +130,6 @@ impl Client { let _ = recv.await.expect("Failed to start listening."); } - // Sync with other peers on the DHT. (GetClosestPeer) - #[allow(dead_code)] - pub async fn random_walk(&mut self, key: Option) -> QueryId { - let (sender, recv) = oneshot::channel(); - - self.client_tx - .send(ClientEvent::RandomWalk { key, sender }) - .await - .unwrap(); - - // Check if the Bootstrap was processed, properly. - recv.await.expect("Failed to Random Walk.") - } - // Bootstrap pub async fn bootstrap(&mut self) -> QueryId { let (sender, recv) = oneshot::channel(); @@ -192,6 +144,47 @@ impl Client { } } +pub(super) enum ClientEvent { + // Event for adding a listening address. + Listen { + addr: Multiaddr, + sender: oneshot::Sender, + }, + // List all known peers. + // TODO: We are using for testing. + #[allow(dead_code)] + KnownPeers { + sender: oneshot::Sender>, + }, + // Dial another peer. + // TODO: We are using for testing. + #[allow(dead_code)] + Dial { + addr: Multiaddr, + peer: PeerId, + sender: oneshot::Sender, + }, + // Bootstrap during the initial connection to the DHT. + // NOTE: All the bootstrap nodes must already be connected to the swarm before we can start the + // bootstrap process. + Bootstrap { + sender: oneshot::Sender, + }, + // Get all listening addresses. + // TODO: We are using for testing. + #[allow(dead_code)] + Listeners { + sender: oneshot::Sender>, + }, + // Read Kademlia Query Result. + // TODO: We are using for testing. + #[allow(dead_code)] + QueryResult { + qid: QueryId, + sender: oneshot::Sender, + }, +} + pub(super) fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { match event { ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { @@ -203,20 +196,6 @@ pub(super) fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) sender.send(qid).unwrap(); } } - ClientEvent::RandomWalk { sender, key } => { - let key = match key { - Some(peerid) => peerid, - None => PeerId::random(), - }; - - let qid = eventloop - .swarm - .behaviour_mut() - .kademlia - .get_closest_peers(key); - - sender.send(qid).unwrap(); - } ClientEvent::KnownPeers { sender } => { let mut result = Vec::new(); diff --git a/crates/spartan-farmer/src/dht/eventloop.rs b/crates/spartan-farmer/src/dht/eventloop.rs index 6e62ccb557..7cb1d6de36 100644 --- a/crates/spartan-farmer/src/dht/eventloop.rs +++ b/crates/spartan-farmer/src/dht/eventloop.rs @@ -57,15 +57,6 @@ impl EventLoop { } KademliaEvent::OutboundQueryCompleted { id, result, .. } => { match &result { - QueryResult::GetClosestPeers(rnd_walk_result) => match rnd_walk_result { - Ok(res) => { - info!( - "Kademlia Random Walk finished successfully: {:?}", - res.peers - ); - } - Err(e) => info!("{:?}", e), - }, QueryResult::Bootstrap(bootstrap_result) => match bootstrap_result { Ok(_res) => info!("Bootstrapping finished successfully."), Err(e) => info!("{:?}", e), From f1deb6889822b38c2d6556e0b8d80824d8534f9f Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Fri, 24 Sep 2021 18:34:35 +0530 Subject: [PATCH 50/52] Remove `ClientType` enum. --- crates/spartan-farmer/src/commands/farm.rs | 11 +---------- crates/spartan-farmer/src/dht.rs | 2 +- crates/spartan-farmer/src/dht/client.rs | 19 ++++--------------- crates/spartan-farmer/src/dht/test.rs | 3 +-- crates/spartan-farmer/src/main.rs | 5 ----- 5 files changed, 7 insertions(+), 33 deletions(-) diff --git a/crates/spartan-farmer/src/commands/farm.rs b/crates/spartan-farmer/src/commands/farm.rs index fce4c4d917..2b2144ae9b 100644 --- a/crates/spartan-farmer/src/commands/farm.rs +++ b/crates/spartan-farmer/src/commands/farm.rs @@ -1,4 +1,4 @@ -use crate::dht::{create_connection, ClientConfig, ClientType}; +use crate::dht::{create_connection, ClientConfig}; use crate::plot::Plot; use crate::{crypto, Salt, Tag, PRIME_SIZE_BYTES, SIGNING_CONTEXT}; use async_std::task; @@ -57,7 +57,6 @@ struct SlotInfo { /// address. pub(crate) async fn farm( listen_addr: Option, - bootstrap: bool, bootstrap_nodes: Vec, path: PathBuf, ws_server: &str, @@ -85,16 +84,8 @@ pub(crate) async fn farm( // 2. Put the swarm in its own task. // 3. The task will run an eventloop and keep discovering new peers. - let client_type = if bootstrap { - info!("I'm a bootstrap node."); - ClientType::Bootstrap - } else { - ClientType::Normal - }; - let config = ClientConfig { bootstrap_nodes, - client_type, listen_addr, }; diff --git a/crates/spartan-farmer/src/dht.rs b/crates/spartan-farmer/src/dht.rs index df216ad67a..d217e73e57 100644 --- a/crates/spartan-farmer/src/dht.rs +++ b/crates/spartan-farmer/src/dht.rs @@ -11,4 +11,4 @@ mod eventloop; #[cfg(test)] mod test; -pub(crate) use client::{create_connection, ClientConfig, ClientType}; +pub(crate) use client::{create_connection, ClientConfig}; diff --git a/crates/spartan-farmer/src/dht/client.rs b/crates/spartan-farmer/src/dht/client.rs index 5d9f91e1c5..e46188c968 100644 --- a/crates/spartan-farmer/src/dht/client.rs +++ b/crates/spartan-farmer/src/dht/client.rs @@ -14,24 +14,13 @@ use futures::prelude::*; type OneshotError = Box; type OneshotType = Result<(), OneshotError>; -use super::{core::create_node, eventloop::EventLoop}; -use libp2p::kad::QueryInfo; - -#[derive(Copy, Clone, Debug)] -pub enum ClientType { - // Bootstrap node. It uses the following fields from `ClientConfig`: - // 1. `bootstrap_keys`: Private keys/private key location to create bootstrap node peerId. - // 2. `listen_addr`: Listening address for Bootstrap node. - Bootstrap, - // Normal node. It uses the following fields from `ClientConfig`: - // 1. `bootstrap_nodes`: Bootstrap nodes addresses that the normal node must connect to. - // For setting listening address, use client.start_listening. - Normal, -} +use super::{ + core::create_node, + eventloop::{ClientEvent, EventLoop}, +}; pub struct ClientConfig { pub bootstrap_nodes: Vec, // Vec<(Multiaddr, PeerId)>, - pub client_type: ClientType, pub listen_addr: Option, } diff --git a/crates/spartan-farmer/src/dht/test.rs b/crates/spartan-farmer/src/dht/test.rs index 1bc35c867e..31ccc2a642 100644 --- a/crates/spartan-farmer/src/dht/test.rs +++ b/crates/spartan-farmer/src/dht/test.rs @@ -1,6 +1,6 @@ use super::*; use client as dht; -use client::{ClientConfig, ClientType}; +use client::ClientConfig; #[tokio::test] async fn bootstrap_working() { @@ -14,7 +14,6 @@ async fn bootstrap_working() { while clients.len() < 5 { let config = ClientConfig { bootstrap_nodes: Default::default(), - client_type: ClientType::Normal, listen_addr: None, }; diff --git a/crates/spartan-farmer/src/main.rs b/crates/spartan-farmer/src/main.rs index 0340309d93..f823a85349 100644 --- a/crates/spartan-farmer/src/main.rs +++ b/crates/spartan-farmer/src/main.rs @@ -68,9 +68,6 @@ enum Command { /// List of bootstrap nodes to connect to with. #[clap(long)] bootstrap_nodes: Vec, - /// Set this peer as bootstrap node. - #[clap(long, short)] - bootstrap: bool, /// Listening address for P2P peer. #[clap(long)] listen_addr: Option, @@ -109,7 +106,6 @@ fn main() { Command::Farm { custom_path, ws_server, - bootstrap, bootstrap_nodes, listen_addr, } => { @@ -118,7 +114,6 @@ fn main() { runtime .block_on(commands::farm( listen_addr, - bootstrap, bootstrap_nodes, path, &ws_server, From 3770fa08b6ae4650855f303e73fd9034ad9ea770 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Fri, 24 Sep 2021 19:04:38 +0530 Subject: [PATCH 51/52] Move `ClientEvent` and `handle_client_event` to eventloop.rs --- crates/spartan-farmer/src/dht/client.rs | 156 +-------------------- crates/spartan-farmer/src/dht/eventloop.rs | 147 +++++++++++++++++-- 2 files changed, 145 insertions(+), 158 deletions(-) diff --git a/crates/spartan-farmer/src/dht/client.rs b/crates/spartan-farmer/src/dht/client.rs index e46188c968..9c3e952c9e 100644 --- a/crates/spartan-farmer/src/dht/client.rs +++ b/crates/spartan-farmer/src/dht/client.rs @@ -1,23 +1,13 @@ -// Stuff for Kademlia -use libp2p::kad::{QueryId, QueryResult}; - -// Stuff needed to create the swarm -use libp2p::{Multiaddr, PeerId}; - -// Stuff needed to set up channels between Client API task and EventLoop task. +use super::{ + core::create_node, + eventloop::{ClientEvent, EventLoop}, +}; use futures::channel::{ mpsc::{channel, Sender}, oneshot, }; use futures::prelude::*; - -type OneshotError = Box; -type OneshotType = Result<(), OneshotError>; - -use super::{ - core::create_node, - eventloop::{ClientEvent, EventLoop}, -}; +use libp2p::{kad::QueryId, Multiaddr, PeerId}; pub struct ClientConfig { pub bootstrap_nodes: Vec, // Vec<(Multiaddr, PeerId)>, @@ -133,145 +123,11 @@ impl Client { } } -pub(super) enum ClientEvent { - // Event for adding a listening address. - Listen { - addr: Multiaddr, - sender: oneshot::Sender, - }, - // List all known peers. - // TODO: We are using for testing. - #[allow(dead_code)] - KnownPeers { - sender: oneshot::Sender>, - }, - // Dial another peer. - // TODO: We are using for testing. - #[allow(dead_code)] - Dial { - addr: Multiaddr, - peer: PeerId, - sender: oneshot::Sender, - }, - // Bootstrap during the initial connection to the DHT. - // NOTE: All the bootstrap nodes must already be connected to the swarm before we can start the - // bootstrap process. - Bootstrap { - sender: oneshot::Sender, - }, - // Get all listening addresses. - // TODO: We are using for testing. - #[allow(dead_code)] - Listeners { - sender: oneshot::Sender>, - }, - // Read Kademlia Query Result. - // TODO: We are using for testing. - #[allow(dead_code)] - QueryResult { - qid: QueryId, - sender: oneshot::Sender, - }, -} - -pub(super) fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { - match event { - ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { - Ok(_) => sender.send(Ok(())).unwrap(), - Err(e) => sender.send(Err(Box::new(e))).unwrap(), - }, - ClientEvent::Bootstrap { sender } => { - if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { - sender.send(qid).unwrap(); - } - } - ClientEvent::KnownPeers { sender } => { - let mut result = Vec::new(); - - for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { - for record in bucket.iter() { - result.push(*record.node.key.preimage()); - } - } - - sender.send(result).unwrap(); - } - ClientEvent::Dial { addr, peer, sender } => { - eventloop - .swarm - .behaviour_mut() - .kademlia - .add_address(&peer, addr.clone()); - - eventloop.swarm.dial_addr(addr).unwrap(); - - sender.send(Ok(())).unwrap(); - } - ClientEvent::Listeners { sender } => { - sender - .send(eventloop.swarm.listeners().cloned().collect::>()) - .unwrap(); - } - ClientEvent::QueryResult { qid, sender } => { - if eventloop.query_result.contains_key(&qid) { - let result = match eventloop.query_result.remove(&qid).unwrap() { - QueryResult::Bootstrap(result) => match result { - Ok(result) => format!( - "[RESULT] This query still has {:?} peers remaining.", - result.num_remaining - ), - Err(e) => format!("{:?}", e), - }, - QueryResult::GetClosestPeers(result) => match result { - Ok(result) => { - format!("This query produced {:?} peers.", result.peers.len()) - } - Err(e) => format!("{:?}", e), - }, - _ => "Unknown QueryResult Type".to_string(), - }; - - sender.send(result).unwrap(); - } else { - let query = eventloop - .swarm - .behaviour_mut() - .kademlia - .query(&qid) - .unwrap(); - - let stats = format!( - "Total Requests: {}\nFailed: {}\nSucceded: {}\nPending: {}\n", - query.stats().num_requests(), - query.stats().num_failures(), - query.stats().num_successes(), - query.stats().num_pending() - ); - - let info = match query.info() { - QueryInfo::Bootstrap { remaining, .. } => { - format!( - "[INFO] This query still has {:?} peers remaining.", - remaining - ) - } - _ => "Unknown QueryInfo Type".to_string(), - }; - - sender.send(stats + &info).unwrap(); - } - } - } -} - // This method will construct a new Swarm and EventLoop object. pub fn create_connection(config: &ClientConfig) -> (Client, EventLoop) { let (client_tx, client_rx) = channel(10); - let (peerid, swarm) = match config.client_type { - ClientType::Bootstrap => create_node(config), - ClientType::Normal => create_node(config), - }; + let (peerid, swarm) = create_node(config); let eventloop = EventLoop::new(swarm, client_rx); let client = Client::new(peerid, client_tx); diff --git a/crates/spartan-farmer/src/dht/eventloop.rs b/crates/spartan-farmer/src/dht/eventloop.rs index 7cb1d6de36..475fd89e55 100644 --- a/crates/spartan-farmer/src/dht/eventloop.rs +++ b/crates/spartan-farmer/src/dht/eventloop.rs @@ -1,20 +1,20 @@ // Stuff for Kademlia -use libp2p::kad::{KademliaEvent, QueryId, QueryResult}; - +use libp2p::kad::{KademliaEvent, QueryId, QueryInfo, QueryResult}; use libp2p::{swarm::SwarmEvent, Swarm}; +use libp2p::{Multiaddr, PeerId}; // Stuff needed to set up channels between Client API task and EventLoop task. use futures::channel::mpsc::Receiver; +use futures::channel::oneshot; use futures::StreamExt; - use log::info; - -use super::{ - client::{handle_client_event, ClientEvent}, - core::{ComposedBehaviour, ComposedEvent}, -}; use std::collections::HashMap; +use super::core::{ComposedBehaviour, ComposedEvent}; + +type OneshotError = Box; +type OneshotType = Result<(), OneshotError>; + pub struct EventLoop { pub(super) swarm: Swarm, // Channel to receive events from Client. @@ -78,3 +78,134 @@ impl EventLoop { } } } + +pub(super) enum ClientEvent { + // Event for adding a listening address. + Listen { + addr: Multiaddr, + sender: oneshot::Sender, + }, + // List all known peers. + // TODO: We are using for testing. + #[allow(dead_code)] + KnownPeers { + sender: oneshot::Sender>, + }, + // Dial another peer. + // TODO: We are using for testing. + #[allow(dead_code)] + Dial { + addr: Multiaddr, + peer: PeerId, + sender: oneshot::Sender, + }, + // Bootstrap during the initial connection to the DHT. + // NOTE: All the bootstrap nodes must already be connected to the swarm before we can start the + // bootstrap process. + Bootstrap { + sender: oneshot::Sender, + }, + // Get all listening addresses. + // TODO: We are using for testing. + #[allow(dead_code)] + Listeners { + sender: oneshot::Sender>, + }, + // Read Kademlia Query Result. + // TODO: We are using for testing. + #[allow(dead_code)] + QueryResult { + qid: QueryId, + sender: oneshot::Sender, + }, +} + +pub(super) fn handle_client_event(eventloop: &mut EventLoop, event: ClientEvent) { + match event { + ClientEvent::Listen { addr, sender } => match eventloop.swarm.listen_on(addr) { + Ok(_) => sender.send(Ok(())).unwrap(), + Err(e) => sender.send(Err(Box::new(e))).unwrap(), + }, + ClientEvent::Bootstrap { sender } => { + if let Ok(qid) = eventloop.swarm.behaviour_mut().kademlia.bootstrap() { + sender.send(qid).unwrap(); + } + } + ClientEvent::KnownPeers { sender } => { + let mut result = Vec::new(); + + for bucket in eventloop.swarm.behaviour_mut().kademlia.kbuckets() { + for record in bucket.iter() { + result.push(*record.node.key.preimage()); + } + } + + sender.send(result).unwrap(); + } + ClientEvent::Dial { addr, peer, sender } => { + eventloop + .swarm + .behaviour_mut() + .kademlia + .add_address(&peer, addr.clone()); + + eventloop.swarm.dial_addr(addr).unwrap(); + + sender.send(Ok(())).unwrap(); + } + ClientEvent::Listeners { sender } => { + sender + .send(eventloop.swarm.listeners().cloned().collect::>()) + .unwrap(); + } + ClientEvent::QueryResult { qid, sender } => { + if eventloop.query_result.contains_key(&qid) { + let result = match eventloop.query_result.remove(&qid).unwrap() { + QueryResult::Bootstrap(result) => match result { + Ok(result) => format!( + "[RESULT] This query still has {:?} peers remaining.", + result.num_remaining + ), + Err(e) => format!("{:?}", e), + }, + QueryResult::GetClosestPeers(result) => match result { + Ok(result) => { + format!("This query produced {:?} peers.", result.peers.len()) + } + Err(e) => format!("{:?}", e), + }, + _ => "Unknown QueryResult Type".to_string(), + }; + + sender.send(result).unwrap(); + } else { + let query = eventloop + .swarm + .behaviour_mut() + .kademlia + .query(&qid) + .unwrap(); + + let stats = format!( + "Total Requests: {}\nFailed: {}\nSucceded: {}\nPending: {}\n", + query.stats().num_requests(), + query.stats().num_failures(), + query.stats().num_successes(), + query.stats().num_pending() + ); + + let info = match query.info() { + QueryInfo::Bootstrap { remaining, .. } => { + format!( + "[INFO] This query still has {:?} peers remaining.", + remaining + ) + } + _ => "Unknown QueryInfo Type".to_string(), + }; + + sender.send(stats + &info).unwrap(); + } + } + } +} From 408b3892ab204d133da2f76dfc29321615c21887 Mon Sep 17 00:00:00 2001 From: Tejas Sanap Date: Mon, 27 Sep 2021 00:47:15 +0530 Subject: [PATCH 52/52] Fix gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e53981c959..5bb2fd7b8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.gitignore /.idea /target -.vscode/ +/.vscode