Upgrade to tokio 0.3 (#1839)

## Description This PR updates Lighthouse to tokio 0.3. It includes a number of dependency updates and some structural changes as to how we create and spawn tasks. This also brings with it a number of various improvements: - Discv5 update - Libp2p update - Fix for recompilation issues - Improved UPnP port mapping handling - Futures dependency update - Log downgrade to traces for rejecting peers when we've reached our max Co-authored-by: blacktemplar <[email protected]>
sigp · Nov 27, 2020 · db63f18 · db63f18
1 parent 1312844
commit db63f18
Show file tree

Hide file tree

Showing 81 changed files with 3,352 additions and 2,454 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Makefile b/Makefile
@@ -140,7 +140,7 @@ audit:
 	#
 	# Tracking issue:
 	# https://github.com/sigp/lighthouse/issues/1669
-	cargo audit --ignore RUSTSEC-2020-0043
+	cargo audit --ignore RUSTSEC-2020-0043 --ignore RUSTSEC-2016-0002 --ignore RUSTSEC-2020-0008 --ignore RUSTSEC-2017-0002
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:

diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml
@@ -20,20 +20,21 @@ eth2_ssz_derive = "0.1.0"
 hex = "0.4.2"
 rayon = "1.4.1"
 eth2_testnet_config = { path = "../common/eth2_testnet_config" }
-futures = { version = "0.3.5", features = ["compat"] }
+futures = { version = "0.3.7", features = ["compat"] }
 clap_utils = { path = "../common/clap_utils" }
 directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
 rand = "0.7.3"
 validator_dir = { path = "../common/validator_dir" }
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 eth2_keystore = { path = "../crypto/eth2_keystore" }
 account_utils = { path = "../common/account_utils" }
 slashing_protection = { path = "../validator_client/slashing_protection" }
 eth2 = {path = "../common/eth2"}
 safe_arith = {path = "../consensus/safe_arith"}
 slot_clock = { path = "../common/slot_clock" }
+tokio-compat-02 = "0.1"
 
 [dev-dependencies]
 tempfile = "3.1.0"
diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs
@@ -12,6 +12,7 @@ use safe_arith::SafeArith;
 use slot_clock::{SlotClock, SystemTimeSlotClock};
 use std::path::PathBuf;
 use std::time::Duration;
+use tokio_compat_02::FutureExt;
 use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
 
 pub const CMD: &str = "exit";
@@ -58,7 +59,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run<E: EthSpec>(matches: &ArgMatches, mut env: Environment<E>) -> Result<(), String> {
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(), String> {
     let keystore_path: PathBuf = clap_utils::parse_required(matches, KEYSTORE_FLAG)?;
     let password_file_path: Option<PathBuf> =
         clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?;
@@ -76,14 +77,17 @@ pub fn cli_run<E: EthSpec>(matches: &ArgMatches, mut env: Environment<E>) -> Res
         .clone()
         .expect("network should have a valid config");
 
-    env.runtime().block_on(publish_voluntary_exit::<E>(
-        &keystore_path,
-        password_file_path.as_ref(),
-        &client,
-        &spec,
-        stdin_inputs,
-        &testnet_config,
-    ))?;
+    env.runtime().block_on(
+        publish_voluntary_exit::<E>(
+            &keystore_path,
+            password_file_path.as_ref(),
+            &client,
+            &spec,
+            stdin_inputs,
+            &testnet_config,
+        )
+        .compat(),
+    )?;
 
     Ok(())
 }
@@ -155,7 +159,7 @@ async fn publish_voluntary_exit<E: EthSpec>(
             .post_beacon_pool_voluntary_exits(&signed_voluntary_exit)
             .await
             .map_err(|e| format!("Failed to publish voluntary exit: {}", e))?;
-        tokio::time::delay_for(std::time::Duration::from_secs(1)).await; // Provides nicer UX.
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await; // Provides nicer UX.
         eprintln!(
             "Successfully validated and published voluntary exit for validator {}",
             keypair.pk

diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml
@@ -10,6 +10,7 @@ path = "src/lib.rs"
 
 [dev-dependencies]
 node_test_rig = { path = "../testing/node_test_rig" }
+tokio-compat-02 = "0.1"
 
 [features]
 write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
@@ -26,12 +27,12 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_tr
 slog-term = "2.6.0"
 slog-async = "2.5.0"
 ctrlc = { version = "3.1.6", features = ["termination"] }
-tokio = { version = "0.2.22", features = ["time"] }
+tokio = { version = "0.3.2", features = ["time"] }
 exit-future = "0.2.0"
 dirs = "3.0.1"
 logging = { path = "../common/logging" }
 directory = {path = "../common/directory"}
-futures = "0.3.5"
+futures = "0.3.7"
 environment = { path = "../lighthouse/environment" }
 task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }

diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml
@@ -40,10 +40,10 @@ eth2_ssz_derive = "0.1.0"
 state_processing = { path = "../../consensus/state_processing" }
 tree_hash = "0.1.1"
 types = { path = "../../consensus/types" }
-tokio = "0.2.22"
+tokio = "0.3.2"
 eth1 = { path = "../eth1" }
 websocket_server = { path = "../websocket_server" }
-futures = "0.3.5"
+futures = "0.3.7"
 genesis = { path = "../genesis" }
 integer-sqrt = "0.1.5"
 rand = "0.7.3"

diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs
@@ -413,7 +413,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
 
         // Ensure there has been no other observed aggregate for the given `aggregator_index`.
         //
-        // Note: do not observe yet, only observe once the attestation has been verfied.
+        // Note: do not observe yet, only observe once the attestation has been verified.
         match chain
             .observed_aggregators
             .read()

diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml
@@ -27,9 +27,9 @@ error-chain = "0.12.4"
 serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 slog-async = "2.5.0"
-tokio = "0.2.22"
+tokio = "0.3.2"
 dirs = "3.0.1"
-futures = "0.3.5"
+futures = "0.3.7"
 reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }

diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs
@@ -254,10 +254,16 @@ where
                     let (listen_addr, server) = http_api::serve(ctx, exit_future)
                         .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
 
+                    let log_clone = context.log().clone();
+                    let http_api_task = async move {
+                        server.await;
+                        debug!(log_clone, "HTTP API server task ended");
+                    };
+
                     context
                         .clone()
                         .executor
-                        .spawn_without_exit(async move { server.await }, "http-api");
+                        .spawn_without_exit(http_api_task, "http-api");
 
                     Some(listen_addr)
                 } else {
@@ -283,7 +289,7 @@ where
                             "Waiting for HTTP server port to open";
                             "port" => http_listen
                         );
-                        tokio::time::delay_for(Duration::from_secs(1)).await;
+                        tokio::time::sleep(Duration::from_secs(1)).await;
                     }
                 }
 
@@ -442,10 +448,16 @@ where
             let (listen_addr, server) = http_api::serve(ctx, exit)
                 .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
 
+            let http_log = runtime_context.log().clone();
+            let http_api_task = async move {
+                server.await;
+                debug!(http_log, "HTTP API server task ended");
+            };
+
             runtime_context
                 .clone()
                 .executor
-                .spawn_without_exit(async move { server.await }, "http-api");
+                .spawn_without_exit(http_api_task, "http-api");
 
             Some(listen_addr)
         } else {

diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs
@@ -7,7 +7,7 @@ use slog::{debug, error, info, warn};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use types::{EthSpec, Slot};
 
 /// Create a warning log whenever the peer count is at or below this value.
@@ -56,7 +56,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         "peers" => peer_count_pretty(network.connected_peers()),
                         "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
                     );
-                    delay_for(slot_duration).await;
+                    sleep(slot_duration).await;
                 }
                 _ => break,
             }

diff --git a/beacon_node/eth1/Cargo.toml b/beacon_node/eth1/Cargo.toml
@@ -10,10 +10,11 @@ toml = "0.5.6"
 web3 = "0.11.0"
 sloggers = "1.0.1"
 environment = { path = "../../lighthouse/environment" }
+tokio-compat-02 = "0.1"
 
 [dependencies]
 reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
-futures = { version = "0.3.5", features = ["compat"] }
+futures = { version = "0.3.7", features = ["compat"] }
 serde_json = "1.0.58"
 serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
@@ -25,7 +26,7 @@ tree_hash = "0.1.1"
 eth2_hashing = "0.1.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
 libflate = "1.0.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}