From c9333bdc463b7f92214325a9c6a2d4f31a583486 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 24 May 2021 08:55:21 +0200 Subject: [PATCH 01/52] make remote-ext work with ws and safe RPCs --- Cargo.lock | 58 ++++- utils/frame/remote-externalities/Cargo.toml | 1 + utils/frame/remote-externalities/src/lib.rs | 232 +++++++++++++++----- 3 files changed, 225 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9019acfcbb239..df64b00654653 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,19 @@ version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" +[[package]] +name = "async-tls" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f23d769dbf1838d5df5156e7b1ad404f4c463d1ac2c6aeb6cd943630f8a8400" +dependencies = [ + "futures-core", + "futures-io", + "rustls 0.19.0", + "webpki 0.21.4", + "webpki-roots", +] + [[package]] name = "async-trait" version = "0.1.48" @@ -2103,7 +2116,7 @@ checksum = "3a1387e07917c711fb4ee4f48ea0adb04a3c9739e53ef85bf43ae1edc2937a8b" dependencies = [ "futures-io", "rustls 0.19.0", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -2579,7 +2592,7 @@ dependencies = [ "rustls-native-certs", "tokio 0.2.25", "tokio-rustls", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -2969,6 +2982,28 @@ dependencies = [ "jsonrpsee-types", ] +[[package]] +name = "jsonrpsee-ws-client" +version = "0.2.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6fdb4390bd25358c62e8b778652a564a1723ba07dca0feb3da439c2253fe59f" +dependencies = [ + "async-std", + "async-tls", + "async-trait", + "fnv", + "futures 0.3.13", + "jsonrpsee-types", + "log", + "pin-project 1.0.5", + "serde", + "serde_json", + "soketto", + "thiserror", + "url 2.2.1", + "webpki 0.22.0", +] + [[package]] name = "keccak" version = "0.1.0" @@ -6662,6 +6697,7 @@ dependencies = [ "hex-literal", "jsonrpsee-http-client", "jsonrpsee-proc-macros", + "jsonrpsee-ws-client", "log", "parity-scale-codec", "sp-core", @@ -6779,7 +6815,7 @@ dependencies = [ "log", "ring", "sct", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -6792,7 +6828,7 @@ dependencies = [ "log", "ring", "sct", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -10109,7 +10145,7 @@ dependencies = [ "futures-core", "rustls 0.18.1", "tokio 0.2.25", - "webpki", + "webpki 0.21.4", ] [[package]] @@ -11104,13 +11140,23 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki-roots" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" dependencies = [ - "webpki", + "webpki 0.21.4", ] [[package]] diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 7d372e8648eea..362de077d661a 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -14,6 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpsee-http-client = { version = "=0.2.0-alpha.6", default-features = false, features = ["tokio02"] } +jsonrpsee-ws-client = { version = "=0.2.0-alpha.6", default-features = false } jsonrpsee-proc-macros = "=0.2.0-alpha.6" hex-literal = "0.3.1" diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 3ec16ea1982cd..9bb8e4cef3e30 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -113,33 +113,45 @@ use sp_core::{ storage::{StorageKey, StorageData}, }; use codec::{Encode, Decode}; -use jsonrpsee_http_client::{HttpClient, HttpClientBuilder}; - use sp_runtime::traits::Block as BlockT; +use jsonrpsee_ws_client::{WsClientBuilder, WsClient}; type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; -const TARGET: &str = "http://localhost:9933"; +const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; jsonrpsee_proc_macros::rpc_client_api! { RpcApi { - #[rpc(method = "state_getPairs", positional_params)] - fn storage_pairs(prefix: StorageKey, hash: Option) -> Vec<(StorageKey, StorageData)>; - #[rpc(method = "chain_getFinalizedHead")] + #[rpc(method = "state_getStorage", positional_params)] + fn get_storage(prefix: StorageKey, hash: Option) -> StorageData; + #[rpc(method = "state_getKeysPaged", positional_params)] + fn get_keys_paged( + prefix: Option, + count: u32, + start_key: Option, + hash: Option, + ) -> Vec; + #[rpc(method = "chain_getFinalizedHead", positional_params)] fn finalized_head() -> B::Hash; } } /// The execution mode. #[derive(Clone)] -pub enum Mode { +pub enum Mode<'a, B: BlockT> { /// Online. - Online(OnlineConfig), + Online(OnlineConfig<'a, B>), /// Offline. Uses a state snapshot file and needs not any client config. Offline(OfflineConfig), } +impl Default for Mode<'_, B> { + fn default() -> Self { + Mode::Online(OnlineConfig::default()) + } +} + /// configuration of the online execution. /// /// A state snapshot config must be present. @@ -149,32 +161,55 @@ pub struct OfflineConfig { pub state_snapshot: SnapshotConfig, } +/// Description of the transport protocol. +#[derive(Debug)] +pub struct Transport<'a> { + uri: &'a str, + client: Option, +} + +impl Clone for Transport<'_> { + fn clone(&self) -> Self { + Self { uri: self.uri, client: None } + } +} + +impl<'a> From<&'a str> for Transport<'a> { + fn from(t: &'a str) -> Self { + Self { uri: t, client: None } + } +} + /// Configuration of the online execution. /// /// A state snapshot config may be present and will be written to in that case. #[derive(Clone)] -pub struct OnlineConfig { - /// The HTTP uri to use. - pub uri: String, +pub struct OnlineConfig<'a, B: BlockT> { /// The block number at which to connect. Will be latest finalized head if not provided. pub at: Option, /// An optional state snapshot file to WRITE to, not for reading. Not written if set to `None`. pub state_snapshot: Option, /// The modules to scrape. If empty, entire chain state will be scraped. - pub modules: Vec, + pub modules: Vec<&'a str>, + /// Transport config. + pub transport: Transport<'a>, } -impl Default for OnlineConfig { +impl Default for OnlineConfig<'_, B> { fn default() -> Self { - Self { uri: TARGET.to_owned(), at: None, state_snapshot: None, modules: Default::default() } + Self { + transport: Transport { uri: DEFAULT_TARGET, client: None }, + at: None, + state_snapshot: None, + modules: vec![], + } } } -impl OnlineConfig { - /// Return a new http rpc client. - fn rpc(&self) -> HttpClient { - HttpClientBuilder::default().max_request_body_size(u32::MAX).build(&self.uri) - .expect("valid HTTP url; qed") +impl OnlineConfig<'_, B> { + /// Return rpc (ws) client. + fn rpc_client(&self) -> &WsClient { + self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.") } } @@ -198,30 +233,31 @@ impl Default for SnapshotConfig { } /// Builder for remote-externalities. -pub struct Builder { +pub struct Builder<'a, B: BlockT> { + /// Pallets to inject their prefix into the externalities. inject: Vec, - mode: Mode, + /// connectivity mode, online or offline. + mode: Mode<'a, B>, } -impl Default for Builder { +// NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for +// that. +impl Default for Builder<'_, B> { fn default() -> Self { - Self { - inject: Default::default(), - mode: Mode::Online(OnlineConfig::default()), - } + Self { inject: Default::default(), mode: Default::default() } } } // Mode methods -impl Builder { - fn as_online(&self) -> &OnlineConfig { +impl<'a, B: BlockT> Builder<'a, B> { + fn as_online(&self) -> &OnlineConfig<'a, B> { match &self.mode { Mode::Online(config) => &config, _ => panic!("Unexpected mode: Online"), } } - fn as_online_mut(&mut self) -> &mut OnlineConfig { + fn as_online_mut(&mut self) -> &mut OnlineConfig<'a, B> { match &mut self.mode { Mode::Online(config) => config, _ => panic!("Unexpected mode: Online"), @@ -230,33 +266,99 @@ impl Builder { } // RPC methods -impl Builder { +impl Builder<'_, B> { async fn rpc_get_head(&self) -> Result { trace!(target: LOG_TARGET, "rpc: finalized_head"); - RpcApi::::finalized_head(&self.as_online().rpc()).await.map_err(|e| { + RpcApi::::finalized_head(self.as_online().rpc_client()).await.map_err(|e| { error!("Error = {:?}", e); "rpc finalized_head failed." }) } - /// Relay the request to `state_getPairs` rpc endpoint. + /// Get all the keys at `prefix` at `hash` using the paged, safe RPC methods. + async fn get_keys_paged( + &self, + prefix: StorageKey, + hash: B::Hash, + ) -> Result, &'static str> { + const PAGE: u32 = 512; + let mut last_key: Option = None; + let mut all_keys: Vec = vec![]; + let keys = loop { + let page = RpcApi::::get_keys_paged( + self.as_online().rpc_client(), + Some(prefix.clone()), + PAGE, + last_key.clone(), + Some(hash), + ) + .await + .map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc get_keys failed" + })?; + all_keys.extend(page.clone()); + + if page.len() < (PAGE as usize) { + debug!(target: LOG_TARGET, "last page received: {}", page.len()); + break all_keys; + } else { + let new_last_key = + all_keys.last().expect("all_keys is populated; has .last(); qed"); + debug!( + target: LOG_TARGET, + "new total = {}, full page received: {:?}", + all_keys.len(), + HexDisplay::from(new_last_key) + ); + last_key = Some(new_last_key.clone()); + } + }; + + Ok(keys) + } + + /// Synonym of `rpc_get_pairs_unsafe` that uses paged queries to first get the keys, and then + /// map them to values one by one. /// - /// Note that this is an unsafe RPC. - async fn rpc_get_pairs( + /// This can work with public nodes. But, expect it to be darn slow. + pub(crate) async fn rpc_get_pairs_paged( &self, prefix: StorageKey, at: B::Hash, ) -> Result, &'static str> { - trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); - RpcApi::::storage_pairs(&self.as_online().rpc(), prefix, Some(at)).await.map_err(|e| { - error!("Error = {:?}", e); - "rpc storage_pairs failed" - }) + let keys = self.get_keys_paged(prefix, at).await?; + let keys_count = keys.len(); + info!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); + + let mut key_values: Vec = vec![]; + for key in keys { + let value = + RpcApi::::get_storage(self.as_online().rpc_client(), key.clone(), Some(at)) + .await + .map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc get_storage failed" + })?; + key_values.push((key, value)); + if key_values.len() % 1000 == 0 { + let ratio: f64 = key_values.len() as f64 / keys_count as f64; + debug!( + target: LOG_TARGET, + "progress = {:.2} [{} / {}]", + ratio, + key_values.len(), + keys_count, + ); + } + } + + Ok(key_values) } } // Internal methods -impl Builder { +impl Builder<'_, B> { /// Save the given data as state snapshot. fn save_state_snapshot(&self, data: &[KeyPair], path: &Path) -> Result<(), &'static str> { info!(target: LOG_TARGET, "writing to state snapshot file {:?}", path); @@ -279,13 +381,13 @@ impl Builder { .at .expect("online config must be initialized by this point; qed.") .clone(); - info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at); + info!(target: LOG_TARGET, "scraping keypairs from remote @ {:?}", at); let keys_and_values = if config.modules.len() > 0 { let mut filtered_kv = vec![]; for f in config.modules.iter() { let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); - let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await?; + let module_kv = self.rpc_get_pairs_paged(hashed_prefix.clone(), at).await?; info!( target: LOG_TARGET, "downloaded data for module {} (count: {} / prefix: {:?}).", @@ -298,22 +400,34 @@ impl Builder { filtered_kv } else { info!(target: LOG_TARGET, "downloading data for all modules."); - self.rpc_get_pairs(StorageKey(vec![]), at).await?.into_iter().collect::>() + self.rpc_get_pairs_paged(StorageKey(vec![]), at).await? }; Ok(keys_and_values) } - async fn init_remote_client(&mut self) -> Result<(), &'static str> { - info!(target: LOG_TARGET, "initializing remote client to {:?}", self.as_online().uri); + pub(crate) async fn init_remote_client(&mut self) -> Result<(), &'static str> { + let mut online = self.as_online_mut(); + info!(target: LOG_TARGET, "initializing remote client to {:?}", online.transport.uri); + + // First, initialize the ws client. + let ws_client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(online.transport.uri) + .await + .map_err(|_| "failed to build ws client")?; + online.transport.client = Some(ws_client); + + // Then, if `at` is not set, set it. if self.as_online().at.is_none() { let at = self.rpc_get_head().await?; self.as_online_mut().at = Some(at); } + Ok(()) } - async fn pre_build(mut self) -> Result, &'static str> { + pub(crate) async fn pre_build(mut self) -> Result, &'static str> { let mut base_kv = match self.mode.clone() { Mode::Offline(config) => self.load_state_snapshot(&config.state_snapshot.path)?, Mode::Online(config) => { @@ -337,7 +451,7 @@ impl Builder { } // Public methods -impl Builder { +impl<'a, B: BlockT> Builder<'a, B> { /// Create a new builder. pub fn new() -> Self { Default::default() @@ -352,7 +466,7 @@ impl Builder { } /// Configure a state snapshot to be used. - pub fn mode(mut self, mode: Mode) -> Self { + pub fn mode(mut self, mode: Mode<'a, B>) -> Self { self.mode = mode; self } @@ -380,8 +494,9 @@ mod test_prelude { pub(crate) fn init_logger() { let _ = env_logger::Builder::from_default_env() - .format_module_path(false) + .format_module_path(true) .format_level(true) + .filter_module(LOG_TARGET, log::LevelFilter::Debug) .try_init(); } } @@ -395,7 +510,7 @@ mod tests { init_logger(); Builder::::new() .mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig { path: "test_data/proxy_test".into() }, + state_snapshot: SnapshotConfig::new("test_data/proxy_test"), })) .build() .await @@ -411,9 +526,9 @@ mod remote_tests { #[tokio::test] async fn can_build_one_pallet() { init_logger(); - Builder::::new() + Builder::<'_, Block>::new() .mode(Mode::Online(OnlineConfig { - modules: vec!["Proxy".into()], + modules: vec!["Proxy"], ..Default::default() })) .build() @@ -425,21 +540,18 @@ mod remote_tests { #[tokio::test] async fn can_create_state_snapshot() { init_logger(); - Builder::::new() + Builder::<'_, Block>::new() .mode(Mode::Online(OnlineConfig { - state_snapshot: Some(SnapshotConfig { - name: "test_snapshot_to_remove.bin".into(), - ..Default::default() - }), + state_snapshot: Some(SnapshotConfig::new("test_snapshot_to_remove.bin")), + modules: vec!["Balances"], ..Default::default() })) .build() .await .expect("Can't reach the remote node. Is it running?") - .unwrap() .execute_with(|| {}); - let to_delete = std::fs::read_dir(SnapshotConfig::default().directory) + let to_delete = std::fs::read_dir(SnapshotConfig::default().path) .unwrap() .into_iter() .map(|d| d.unwrap()) @@ -454,7 +566,7 @@ mod remote_tests { } #[tokio::test] - async fn can_build_all() { + async fn can_fetch_all() { init_logger(); Builder::::new() .build() From 5a99270812d9f74d7b2904eb06b054163a17e724 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 24 May 2021 08:56:54 +0200 Subject: [PATCH 02/52] Update docs. --- utils/frame/try-runtime/cli/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 9e41a3fd87e77..ffe62e4aac9d8 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -86,7 +86,7 @@ pub enum State { modules: Option>, /// The url to connect to. - #[structopt(default_value = "http://localhost:9933", parse(try_from_str = parse_url))] + #[structopt(default_value = "ws://localhost:9944", parse(try_from_str = parse_url))] url: String, }, } @@ -109,11 +109,11 @@ fn parse_hash(block_number: &str) -> Result { } fn parse_url(s: &str) -> Result { - if s.starts_with("http://") { + if s.starts_with("ws://") || s.starts_with("wss://") { // could use Url crate as well, but lets keep it simple for now. Ok(s.to_string()) } else { - Err("not a valid HTTP url: must start with 'http://'") + Err("not a valid WS(S) url: must start with 'ws://' or 'wss://'") } } From 287126b01a1ca3a9a3afc30a98f73d5401d942b3 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Mon, 24 May 2021 09:58:56 +0200 Subject: [PATCH 03/52] Update utils/frame/remote-externalities/Cargo.toml Co-authored-by: Niklas Adolfsson --- utils/frame/remote-externalities/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 362de077d661a..4f6db60af4dbe 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -13,7 +13,6 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee-http-client = { version = "=0.2.0-alpha.6", default-features = false, features = ["tokio02"] } jsonrpsee-ws-client = { version = "=0.2.0-alpha.6", default-features = false } jsonrpsee-proc-macros = "=0.2.0-alpha.6" From d2b0f48186f5b1c7ec2eb784a8977d1264d1b212 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 24 May 2021 09:59:03 +0200 Subject: [PATCH 04/52] Fix test --- utils/frame/try-runtime/cli/src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index ffe62e4aac9d8..18fc027229b3f 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -166,9 +166,12 @@ impl TryRuntimeCmd { block_at, modules } => Builder::::new().mode(Mode::Online(OnlineConfig { - uri: url.into(), + transport: url[..].into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - modules: modules.clone().unwrap_or_default(), + modules: modules + .as_ref() + .map(|m| m.into_iter().map(|x| x.as_ref()).collect::>()) + .unwrap_or_default(), at: block_at.as_ref() .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, ..Default::default() From ab106bd771a8a74057e7031121ea1b66864a08b0 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 24 May 2021 10:19:07 +0200 Subject: [PATCH 05/52] Update lock file --- Cargo.lock | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df64b00654653..794905619683d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2923,25 +2923,6 @@ dependencies = [ "slab", ] -[[package]] -name = "jsonrpsee-http-client" -version = "0.2.0-alpha.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2737440f37efa10e5ef7beeec43d059d29dc92640978be21fcdcef481a2edb0d" -dependencies = [ - "async-trait", - "fnv", - "hyper 0.13.10", - "hyper-rustls", - "jsonrpsee-types", - "jsonrpsee-utils", - "log", - "serde", - "serde_json", - "thiserror", - "url 2.2.1", -] - [[package]] name = "jsonrpsee-proc-macros" version = "0.2.0-alpha.6" @@ -2971,17 +2952,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "jsonrpsee-utils" -version = "0.2.0-alpha.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d63cf4d423614e71fd144a8691208539d2b23d8373e069e2fbe023c5eba5e922" -dependencies = [ - "futures-util", - "hyper 0.13.10", - "jsonrpsee-types", -] - [[package]] name = "jsonrpsee-ws-client" version = "0.2.0-alpha.6" @@ -6695,7 +6665,6 @@ version = "0.9.0" dependencies = [ "env_logger 0.8.3", "hex-literal", - "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-ws-client", "log", From 0b3415d741cb5dfdda227083378d862e79640b51 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 25 May 2021 07:34:33 +0200 Subject: [PATCH 06/52] Update utils/frame/remote-externalities/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- utils/frame/remote-externalities/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 9bb8e4cef3e30..2956dde846cde 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -297,9 +297,10 @@ impl Builder<'_, B> { error!(target: LOG_TARGET, "Error = {:?}", e); "rpc get_keys failed" })?; - all_keys.extend(page.clone()); + let page_len = page.len(); + all_keys.extend(page); - if page.len() < (PAGE as usize) { + if page_len < PAGE as usize { debug!(target: LOG_TARGET, "last page received: {}", page.len()); break all_keys; } else { From 22959b0bd0b5441f69d4029611ecf7de944b24a7 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 25 May 2021 08:01:36 +0200 Subject: [PATCH 07/52] Fix build again. --- utils/frame/remote-externalities/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 2956dde846cde..5b4b20e881700 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -301,7 +301,7 @@ impl Builder<'_, B> { all_keys.extend(page); if page_len < PAGE as usize { - debug!(target: LOG_TARGET, "last page received: {}", page.len()); + debug!(target: LOG_TARGET, "last page received: {}", page_len); break all_keys; } else { let new_last_key = From e7bf17d478ba03407898e2a4df29c1432778a10e Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 25 May 2021 11:04:00 +0200 Subject: [PATCH 08/52] checkpoint, merging the paged rpc now --- Cargo.lock | 3 + bin/node/cli/src/cli.rs | 3 +- utils/frame/remote-externalities/Cargo.toml | 3 + utils/frame/remote-externalities/src/lib.rs | 21 +- utils/frame/try-runtime/cli/Cargo.toml | 1 + utils/frame/try-runtime/cli/src/lib.rs | 298 ++++++++++++++------ 6 files changed, 233 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9019acfcbb239..c07724cf5a1a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6664,6 +6664,8 @@ dependencies = [ "jsonrpsee-proc-macros", "log", "parity-scale-codec", + "serde", + "serde_json", "sp-core", "sp-io", "sp-runtime", @@ -10437,6 +10439,7 @@ dependencies = [ "sc-client-api", "sc-executor", "sc-service", + "serde", "sp-api", "sp-blockchain", "sp-core", diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 9b80a3e345290..f8ab878dd8211 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,8 +47,7 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// Try some experimental command on the runtime. This includes migration and runtime-upgrade - /// testing. + /// Try some experimental command on the runtime. See `try-runtime --help` for more. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 7d372e8648eea..81e9d61f10b50 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -16,6 +16,9 @@ targets = ["x86_64-unknown-linux-gnu"] jsonrpsee-http-client = { version = "=0.2.0-alpha.6", default-features = false, features = ["tokio02"] } jsonrpsee-proc-macros = "=0.2.0-alpha.6" +serde_json = "1.0" +serde = "1.0.0" + hex-literal = "0.3.1" env_logger = "0.8.2" log = "0.4.11" diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 3ec16ea1982cd..c14b897fbce0b 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -113,14 +113,14 @@ use sp_core::{ storage::{StorageKey, StorageData}, }; use codec::{Encode, Decode}; -use jsonrpsee_http_client::{HttpClient, HttpClientBuilder}; +use jsonrpsee_http_client::{HttpClient, HttpClientBuilder, v2::params::JsonRpcParams}; use sp_runtime::traits::Block as BlockT; type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; -const TARGET: &str = "http://localhost:9933"; +const DEFAULT_URI: &str = "http://localhost:9934"; jsonrpsee_proc_macros::rpc_client_api! { RpcApi { @@ -166,10 +166,24 @@ pub struct OnlineConfig { impl Default for OnlineConfig { fn default() -> Self { - Self { uri: TARGET.to_owned(), at: None, state_snapshot: None, modules: Default::default() } + Self { uri: DEFAULT_URI.to_owned(), at: None, state_snapshot: None, modules: Default::default() } } } +// TODO +pub async fn get_header>(from: S, at: B::Hash) -> B::Header where B::Header: serde::de::DeserializeOwned { + use jsonrpsee_http_client::traits::Client; + let params = vec![serde_json::to_value(at).unwrap()]; + let client = HttpClientBuilder::default().max_request_body_size(u32::MAX).build(from).expect("valid HTTP url; qed"); + client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() +} + +pub async fn get_finalized_head>(from: S) -> B::Hash { + use jsonrpsee_http_client::traits::Client; + let client = HttpClientBuilder::default().max_request_body_size(u32::MAX).build(from).expect("valid HTTP url; qed"); + client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() +} + impl OnlineConfig { /// Return a new http rpc client. fn rpc(&self) -> HttpClient { @@ -231,6 +245,7 @@ impl Builder { // RPC methods impl Builder { + /// Get the latest finalized head. async fn rpc_get_head(&self) -> Result { trace!(target: LOG_TARGET, "rpc: finalized_head"); RpcApi::::finalized_head(&self.as_online().rpc()).await.map_err(|e| { diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 592d0a5b99d27..4dd854a9bcf6f 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -15,6 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.8" parity-scale-codec = { version = "2.0.0" } +serde = "1.0.0" sc-service = { version = "0.9.0", default-features = false, path = "../../../../client/service" } sc-cli = { version = "0.9.0", path = "../../../../client/cli" } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 9e41a3fd87e77..136e03a329ab8 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -15,9 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! `Structopt`-ready struct for `try-runtime`. +//! `Structopt`-ready structs for `try-runtime`. -use parity_scale_codec::Decode; +use parity_scale_codec::{Decode, Encode}; use std::{fmt::Debug, path::PathBuf, str::FromStr}; use sc_service::Configuration; use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; @@ -27,11 +27,29 @@ use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; -/// Various commands to try out the new runtime, over configurable states. -/// -/// For now this only assumes running the `on_runtime_upgrade` hooks. -#[derive(Debug, structopt::StructOpt)] -pub struct TryRuntimeCmd { +#[derive(Debug, Clone, structopt::StructOpt)] +pub enum Command { + OnRuntimeUpgrade(OnRuntimeUpgradeCmd), + OffchainWorker(OffchainWorkerCmd), +} + +#[derive(Debug, Clone, structopt::StructOpt)] +pub struct OnRuntimeUpgradeCmd { + #[structopt(subcommand)] + pub state: State, +} + +#[derive(Debug, Clone, structopt::StructOpt)] +pub struct OffchainWorkerCmd { + #[structopt(short, long, multiple = false, parse(try_from_str = parse_hash))] + pub header_at: String, + + #[structopt(subcommand)] + pub state: State, +} + +#[derive(Debug, Clone, structopt::StructOpt)] +pub struct SharedParams { /// The shared parameters #[allow(missing_docs)] #[structopt(flatten)] @@ -53,17 +71,25 @@ pub struct TryRuntimeCmd { value_name = "METHOD", possible_values = &WasmExecutionMethod::variants(), case_insensitive = true, - default_value = "Interpreted" + default_value = "compiled" )] pub wasm_method: WasmExecutionMethod, +} + +/// Various commands to try out the new runtime, over configurable states. +/// +/// For now this only assumes running the `on_runtime_upgrade` hooks. +#[derive(Debug, Clone, structopt::StructOpt)] +pub struct TryRuntimeCmd { + #[structopt(flatten)] + pub shared: SharedParams, - /// The state to use to run the migration. #[structopt(subcommand)] - pub state: State, + pub command: Command, } -/// The state to use for a migration dry-run. -#[derive(Debug, structopt::StructOpt)] +/// The state to use for a migration dry-run. // TODO: terminology is fucked here. +#[derive(Debug, Clone, structopt::StructOpt)] pub enum State { /// Use a state snapshot as state to run the migration. Snap { @@ -117,103 +143,193 @@ fn parse_url(s: &str) -> Result { } } +async fn on_runtime_upgrade(shared: SharedParams, command: OnRuntimeUpgradeCmd, config: Configuration) -> sc_cli::Result<()> +where + B: BlockT, + B::Hash: FromStr, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, + ExecDispatch: NativeExecutionDispatch + 'static, +{ + let spec = config.chain_spec; + let genesis_storage = spec.build_storage()?; + + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + + let wasm_method = shared.wasm_method; + let execution = shared.execution; + + let mut changes = Default::default(); + // don't really care about these -- use the default values. + let max_runtime_instances = config.max_runtime_instances; + let heap_pages = config.default_heap_pages; + let executor = NativeExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + ); + + let ext = { + use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; + let builder = match command.state { + State::Snap { snapshot_path } => { + Builder::::new().mode(Mode::Offline(OfflineConfig { + state_snapshot: SnapshotConfig::new(snapshot_path), + })) + }, + State::Live { + url, + snapshot_path, + block_at, + modules + } => Builder::::new().mode(Mode::Online(OnlineConfig { + uri: url.into(), + state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), + modules: modules.clone().unwrap_or_default(), + at: block_at.as_ref() + .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, + ..Default::default() + })), + }; + + // inject the code into this ext. + builder.inject(&[(code_key, code)]).build().await? + }; + + let encoded_result = StateMachine::<_, _, NumberFor, _>::new( + &ext.backend, + None, + &mut changes, + &executor, + "TryRuntime_on_runtime_upgrade", + &[], + ext.extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code()?, + sp_core::testing::TaskExecutor::new(), + ) + .execute(execution.into()) + .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; + + let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) + .map_err(|e| format!("failed to decode output due to {:?}", e))?; + log::info!( + "TryRuntime_on_runtime_upgrade executed without errors. Consumed weight = {}, total weight = {} ({})", + weight, + total_weight, + weight as f64 / total_weight as f64 + ); + + Ok(()) +} + +async fn offchain_worker(shared: SharedParams, command: OffchainWorkerCmd, config: Configuration) -> sc_cli::Result<()> +where + B: BlockT, + B::Hash: FromStr, + B::Header: serde::de::DeserializeOwned, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, + ExecDispatch: NativeExecutionDispatch + 'static, +{ + let wasm_method = shared.wasm_method; + let execution = shared.execution; + + let mut changes = Default::default(); + let max_runtime_instances = config.max_runtime_instances; + let heap_pages = config.default_heap_pages; + let executor = NativeExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + ); + + let ext = { + use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; + let builder = match command.state { + State::Snap { snapshot_path } => { + Builder::::new().mode(Mode::Offline(OfflineConfig { + state_snapshot: SnapshotConfig::new(snapshot_path), + })) + }, + State::Live { + url, + snapshot_path, + block_at, + modules + } => Builder::::new().mode(Mode::Online(OnlineConfig { + uri: url.into(), + state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), + modules: modules.clone().unwrap_or_default(), + at: block_at.as_ref() + .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, + ..Default::default() + })), + }; + + builder.build().await? + }; + + let header_hash: B::Hash = command.header_at.parse().unwrap(); + let header = remote_externalities::get_header::("http://localhost:9933", header_hash).await; + + let _ = StateMachine::<_, _, NumberFor, _>::new( + &ext.backend, + None, + &mut changes, + &executor, + "OffchainWorkerApi_offchain_worker", + header.encode().as_ref(), + ext.extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code()?, + sp_core::testing::TaskExecutor::new(), + ) + .execute(execution.into()) + .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; + + log::info!("OffchainWorkerApi_offchain_worker executed without errors."); + Ok(()) +} + impl TryRuntimeCmd { pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> where B: BlockT, + B::Header: serde::de::DeserializeOwned, B::Hash: FromStr, ::Err: Debug, NumberFor: FromStr, as FromStr>::Err: Debug, ExecDispatch: NativeExecutionDispatch + 'static, { - let spec = config.chain_spec; - let genesis_storage = spec.build_storage()?; - - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); - let code_key = StorageKey(well_known_keys::CODE.to_vec()); - - let wasm_method = self.wasm_method; - let execution = self.execution; - - let mut changes = Default::default(); - // don't really care about these -- use the default values. - let max_runtime_instances = config.max_runtime_instances; - let heap_pages = config.default_heap_pages; - let executor = NativeExecutor::::new( - wasm_method.into(), - heap_pages, - max_runtime_instances, - ); - - let ext = { - use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; - let builder = match &self.state { - State::Snap { snapshot_path } => { - Builder::::new().mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig::new(snapshot_path), - })) - }, - State::Live { - url, - snapshot_path, - block_at, - modules - } => Builder::::new().mode(Mode::Online(OnlineConfig { - uri: url.into(), - state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - modules: modules.clone().unwrap_or_default(), - at: block_at.as_ref() - .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, - ..Default::default() - })), - }; - - // inject the code into this ext. - builder.inject(&[(code_key, code)]).build().await? - }; - - let encoded_result = StateMachine::<_, _, NumberFor, _>::new( - &ext.backend, - None, - &mut changes, - &executor, - "TryRuntime_on_runtime_upgrade", - &[], - ext.extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) - .runtime_code()?, - sp_core::testing::TaskExecutor::new(), - ) - .execute(execution.into()) - .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; - - let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) - .map_err(|e| format!("failed to decode output due to {:?}", e))?; - log::info!( - "try-runtime executed without errors. Consumed weight = {}, total weight = {} ({})", - weight, - total_weight, - weight as f64 / total_weight as f64 - ); - - Ok(()) + match &self.command { + Command::OnRuntimeUpgrade(ref cmd) => { + on_runtime_upgrade::(self.shared.clone(), cmd.clone(), config).await + } + Command::OffchainWorker(cmd) => { + offchain_worker::(self.shared.clone(), cmd.clone(), config).await + } + } } } impl CliConfiguration for TryRuntimeCmd { fn shared_params(&self) -> &sc_cli::SharedParams { - &self.shared_params + &self.shared.shared_params } fn chain_id(&self, _is_dev: bool) -> sc_cli::Result { - Ok(match self.shared_params.chain { + Ok(match self.shared.shared_params.chain { Some(ref chain) => chain.clone(), None => "dev".into(), }) From 9724ce796202421ff74104bae57964fb16df578d Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 25 May 2021 11:43:19 +0200 Subject: [PATCH 09/52] revert lifetime stuff --- Cargo.lock | 26 ++++++++- utils/frame/remote-externalities/Cargo.toml | 2 +- utils/frame/remote-externalities/src/lib.rs | 60 ++++++++++----------- utils/frame/try-runtime/cli/src/lib.rs | 7 +-- 4 files changed, 57 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 794905619683d..e4ff69448b232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6672,7 +6672,7 @@ dependencies = [ "sp-core", "sp-io", "sp-runtime", - "tokio 0.2.25", + "tokio 1.6.0", ] [[package]] @@ -9994,10 +9994,21 @@ dependencies = [ "pin-project-lite 0.1.12", "signal-hook-registry", "slab", - "tokio-macros", + "tokio-macros 0.2.6", "winapi 0.3.9", ] +[[package]] +name = "tokio" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" +dependencies = [ + "autocfg", + "pin-project-lite 0.2.6", + "tokio-macros 1.2.0", +] + [[package]] name = "tokio-buf" version = "0.1.1" @@ -10073,6 +10084,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-named-pipes" version = "0.1.0" diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 4f6db60af4dbe..0d6336f60d88f 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -26,7 +26,7 @@ sp-core = { version = "3.0.0", path = "../../../primitives/core" } sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } [dev-dependencies] -tokio = { version = "0.2", features = ["macros"] } +tokio = { version = "1.6.0", features = ["macros", "rt"] } [features] remote-test = [] diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 5b4b20e881700..077892baabf71 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -139,14 +139,14 @@ jsonrpsee_proc_macros::rpc_client_api! { /// The execution mode. #[derive(Clone)] -pub enum Mode<'a, B: BlockT> { +pub enum Mode { /// Online. - Online(OnlineConfig<'a, B>), + Online(OnlineConfig), /// Offline. Uses a state snapshot file and needs not any client config. Offline(OfflineConfig), } -impl Default for Mode<'_, B> { +impl Default for Mode { fn default() -> Self { Mode::Online(OnlineConfig::default()) } @@ -163,19 +163,19 @@ pub struct OfflineConfig { /// Description of the transport protocol. #[derive(Debug)] -pub struct Transport<'a> { - uri: &'a str, +pub struct Transport { + uri: String, client: Option, } -impl Clone for Transport<'_> { +impl Clone for Transport { fn clone(&self) -> Self { - Self { uri: self.uri, client: None } + Self { uri: self.uri.clone(), client: None } } } -impl<'a> From<&'a str> for Transport<'a> { - fn from(t: &'a str) -> Self { +impl From for Transport { + fn from(t: String) -> Self { Self { uri: t, client: None } } } @@ -184,21 +184,21 @@ impl<'a> From<&'a str> for Transport<'a> { /// /// A state snapshot config may be present and will be written to in that case. #[derive(Clone)] -pub struct OnlineConfig<'a, B: BlockT> { +pub struct OnlineConfig { /// The block number at which to connect. Will be latest finalized head if not provided. pub at: Option, /// An optional state snapshot file to WRITE to, not for reading. Not written if set to `None`. pub state_snapshot: Option, /// The modules to scrape. If empty, entire chain state will be scraped. - pub modules: Vec<&'a str>, + pub modules: Vec, /// Transport config. - pub transport: Transport<'a>, + pub transport: Transport, } -impl Default for OnlineConfig<'_, B> { +impl Default for OnlineConfig { fn default() -> Self { Self { - transport: Transport { uri: DEFAULT_TARGET, client: None }, + transport: Transport { uri: DEFAULT_TARGET.to_string(), client: None }, at: None, state_snapshot: None, modules: vec![], @@ -206,7 +206,7 @@ impl Default for OnlineConfig<'_, B> { } } -impl OnlineConfig<'_, B> { +impl OnlineConfig { /// Return rpc (ws) client. fn rpc_client(&self) -> &WsClient { self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.") @@ -233,31 +233,31 @@ impl Default for SnapshotConfig { } /// Builder for remote-externalities. -pub struct Builder<'a, B: BlockT> { +pub struct Builder { /// Pallets to inject their prefix into the externalities. inject: Vec, /// connectivity mode, online or offline. - mode: Mode<'a, B>, + mode: Mode, } // NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for // that. -impl Default for Builder<'_, B> { +impl Default for Builder { fn default() -> Self { Self { inject: Default::default(), mode: Default::default() } } } // Mode methods -impl<'a, B: BlockT> Builder<'a, B> { - fn as_online(&self) -> &OnlineConfig<'a, B> { +impl Builder { + fn as_online(&self) -> &OnlineConfig { match &self.mode { Mode::Online(config) => &config, _ => panic!("Unexpected mode: Online"), } } - fn as_online_mut(&mut self) -> &mut OnlineConfig<'a, B> { + fn as_online_mut(&mut self) -> &mut OnlineConfig { match &mut self.mode { Mode::Online(config) => config, _ => panic!("Unexpected mode: Online"), @@ -266,7 +266,7 @@ impl<'a, B: BlockT> Builder<'a, B> { } // RPC methods -impl Builder<'_, B> { +impl Builder { async fn rpc_get_head(&self) -> Result { trace!(target: LOG_TARGET, "rpc: finalized_head"); RpcApi::::finalized_head(self.as_online().rpc_client()).await.map_err(|e| { @@ -359,7 +359,7 @@ impl Builder<'_, B> { } // Internal methods -impl Builder<'_, B> { +impl Builder { /// Save the given data as state snapshot. fn save_state_snapshot(&self, data: &[KeyPair], path: &Path) -> Result<(), &'static str> { info!(target: LOG_TARGET, "writing to state snapshot file {:?}", path); @@ -414,7 +414,7 @@ impl Builder<'_, B> { // First, initialize the ws client. let ws_client = WsClientBuilder::default() .max_request_body_size(u32::MAX) - .build(online.transport.uri) + .build(&online.transport.uri) .await .map_err(|_| "failed to build ws client")?; online.transport.client = Some(ws_client); @@ -452,7 +452,7 @@ impl Builder<'_, B> { } // Public methods -impl<'a, B: BlockT> Builder<'a, B> { +impl Builder { /// Create a new builder. pub fn new() -> Self { Default::default() @@ -467,7 +467,7 @@ impl<'a, B: BlockT> Builder<'a, B> { } /// Configure a state snapshot to be used. - pub fn mode(mut self, mode: Mode<'a, B>) -> Self { + pub fn mode(mut self, mode: Mode) -> Self { self.mode = mode; self } @@ -527,9 +527,9 @@ mod remote_tests { #[tokio::test] async fn can_build_one_pallet() { init_logger(); - Builder::<'_, Block>::new() + Builder::::new() .mode(Mode::Online(OnlineConfig { - modules: vec!["Proxy"], + modules: vec!["Proxy".to_owned()], ..Default::default() })) .build() @@ -541,10 +541,10 @@ mod remote_tests { #[tokio::test] async fn can_create_state_snapshot() { init_logger(); - Builder::<'_, Block>::new() + Builder::::new() .mode(Mode::Online(OnlineConfig { state_snapshot: Some(SnapshotConfig::new("test_snapshot_to_remove.bin")), - modules: vec!["Balances"], + modules: vec!["Balances".to_owned()], ..Default::default() })) .build() diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 18fc027229b3f..c4adab3ce8f85 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -166,12 +166,9 @@ impl TryRuntimeCmd { block_at, modules } => Builder::::new().mode(Mode::Online(OnlineConfig { - transport: url[..].into(), + transport: url.to_owned().into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - modules: modules - .as_ref() - .map(|m| m.into_iter().map(|x| x.as_ref()).collect::>()) - .unwrap_or_default(), + modules: modules.to_owned().unwrap_or_default(), at: block_at.as_ref() .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, ..Default::default() From 9cc06672436c946ef0cf59c2d91a5f751785c3e2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 26 May 2021 20:07:27 -0700 Subject: [PATCH 10/52] WIP: remote client init not working --- bin/node/cli/src/cli.rs | 1 + utils/frame/remote-externalities/src/lib.rs | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 106 +++++++++++--------- 3 files changed, 60 insertions(+), 49 deletions(-) diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index f8ab878dd8211..ff1007904d87f 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -48,6 +48,7 @@ pub enum Subcommand { Benchmark(frame_benchmarking_cli::BenchmarkCmd), /// Try some experimental command on the runtime. See `try-runtime --help` for more. + /// NOTE: `try-runtime` feature must be enabled. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 9d6c1cda80196..546c4a525673f 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -439,7 +439,7 @@ impl Builder { .max_request_body_size(u32::MAX) .build(&online.transport.uri) .await - .map_err(|_| "failed to build ws client")?; + .map_err(|e| "failed to build ws client")?; online.transport.client = Some(ws_client); // Then, if `at` is not set, set it. diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index b104b40f1e80d..13eac45410411 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -26,6 +26,7 @@ use sc_service::NativeExecutionDispatch; use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; +use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; #[derive(Debug, Clone, structopt::StructOpt)] pub enum Command { @@ -41,7 +42,7 @@ pub struct OnRuntimeUpgradeCmd { #[derive(Debug, Clone, structopt::StructOpt)] pub struct OffchainWorkerCmd { - #[structopt(short, long, multiple = false, parse(try_from_str = parse_hash))] + #[structopt(short, long, multiple = false, parse(try_from_str = util::parse_hash))] pub header_at: String, #[structopt(subcommand)] @@ -76,9 +77,9 @@ pub struct SharedParams { pub wasm_method: WasmExecutionMethod, } -/// Various commands to try out the new runtime, over configurable states. +/// Various commands to try out against runtime state at a specific block. /// -/// For now this only assumes running the `on_runtime_upgrade` hooks. +/// NOTE: For now this only assumes running the `on_runtime_upgrade` hooks. // TOOD wut dis mean? #[derive(Debug, Clone, structopt::StructOpt)] pub struct TryRuntimeCmd { #[structopt(flatten)] @@ -88,23 +89,23 @@ pub struct TryRuntimeCmd { pub command: Command, } -/// The state to use for a migration dry-run. // TODO: terminology is fucked here. +/// The source of runtime state to try operations against. #[derive(Debug, Clone, structopt::StructOpt)] pub enum State { - /// Use a state snapshot as state to run the migration. + /// Use a state snapshot as the source of runtime state. Snap { snapshot_path: PathBuf, }, - /// Use a live chain to run the migration. + /// Use a live chain as the source of runtime state. Live { - /// An optional state snapshot file to WRITE to. Not written if set to `None`. + /// An optional state snapshot file to WRITE to. #[structopt(short, long)] snapshot_path: Option, /// The block hash at which to connect. /// Will be latest finalized head if not provided. - #[structopt(short, long, multiple = false, parse(try_from_str = parse_hash))] + #[structopt(short, long, multiple = false, parse(try_from_str = util::parse_hash))] block_at: Option, /// The modules to scrape. If empty, entire chain state will be scraped. @@ -112,37 +113,11 @@ pub enum State { modules: Option>, /// The url to connect to. - #[structopt(default_value = "ws://localhost:9944", parse(try_from_str = parse_url))] + #[structopt(default_value = "ws://localhost:9944", parse(try_from_str = util::parse_url))] url: String, }, } -fn parse_hash(block_number: &str) -> Result { - let block_number = if block_number.starts_with("0x") { - &block_number[2..] - } else { - block_number - }; - - if let Some(pos) = block_number.chars().position(|c| !c.is_ascii_hexdigit()) { - Err(format!( - "Expected block hash, found illegal hex character at position: {}", - 2 + pos, - )) - } else { - Ok(block_number.into()) - } -} - -fn parse_url(s: &str) -> Result { - if s.starts_with("ws://") || s.starts_with("wss://") { - // could use Url crate as well, but lets keep it simple for now. - Ok(s.to_string()) - } else { - Err("not a valid WS(S) url: must start with 'ws://' or 'wss://'") - } -} - async fn on_runtime_upgrade(shared: SharedParams, command: OnRuntimeUpgradeCmd, config: Configuration) -> sc_cli::Result<()> where B: BlockT, @@ -178,7 +153,6 @@ where ); let ext = { - use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; let builder = match command.state { State::Snap { snapshot_path } => { Builder::::new().mode(Mode::Offline(OfflineConfig { @@ -253,34 +227,42 @@ where max_runtime_instances, ); - let ext = { - use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; - let builder = match command.state { - State::Snap { snapshot_path } => { - Builder::::new().mode(Mode::Offline(OfflineConfig { - state_snapshot: SnapshotConfig::new(snapshot_path), - })) - }, + let (online_config, url) = match command.state { State::Live { url, snapshot_path, block_at, modules - } => Builder::::new().mode(Mode::Online(OnlineConfig { + } => { + let online_config = OnlineConfig { transport: url.to_owned().into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), modules: modules.to_owned().unwrap_or_default(), at: block_at.as_ref() .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, ..Default::default() - })), - }; + }; + + (online_config, url) + }, + _ => { + panic!("Only live state is supported"); + } + }; + + let ext = { + let builder = Builder::::new().mode(Mode::Online(online_config)); builder.build().await? }; let header_hash: B::Hash = command.header_at.parse().unwrap(); - let header = remote_externalities::get_header::("http://localhost:9933", header_hash).await; + // TODO should this be ws to the arg url?? + log::info!("ws url: {}", url); + let header = remote_externalities::get_header::( + url, + header_hash + ).await; let _ = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, @@ -335,3 +317,31 @@ impl CliConfiguration for TryRuntimeCmd { }) } } + +mod util { + pub fn parse_hash(block_number: &str) -> Result { + let block_number = if block_number.starts_with("0x") { + &block_number[2..] + } else { + block_number + }; + + if let Some(pos) = block_number.chars().position(|c| !c.is_ascii_hexdigit()) { + Err(format!( + "Expected block hash, found illegal hex character at position: {}", + 2 + pos, + )) + } else { + Ok(block_number.into()) + } + } + + pub fn parse_url(s: &str) -> Result { + if s.starts_with("ws://") || s.starts_with("wss://") { + // could use Url crate as well, but lets keep it simple for now. + Ok(s.to_string()) + } else { + Err("not a valid WS(S) url: must start with 'ws://' or 'wss://'") + } + } +} From 892e8c550b87db187f2a78f2e826cb652b304a2b Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 26 May 2021 22:07:40 -0700 Subject: [PATCH 11/52] Small cleanups --- utils/frame/remote-externalities/src/lib.rs | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 546c4a525673f..f09348dcc9ec9 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -104,6 +104,7 @@ use std::{ fs, path::{Path, PathBuf}, + error, }; use log::*; use sp_core::hashing::twox_128; @@ -438,7 +439,6 @@ impl Builder { let ws_client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(&online.transport.uri) - .await .map_err(|e| "failed to build ws client")?; online.transport.client = Some(ws_client); diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 13eac45410411..c3ed1af5ab31d 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -257,8 +257,6 @@ where }; let header_hash: B::Hash = command.header_at.parse().unwrap(); - // TODO should this be ws to the arg url?? - log::info!("ws url: {}", url); let header = remote_externalities::get_header::( url, header_hash From fe9ced0c6048b45857e8806f8780a9a9a686421e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 27 May 2021 11:07:42 -0700 Subject: [PATCH 12/52] use jsonrpsee alpha.7 --- Cargo.lock | 350 +++++++++++++------- utils/frame/remote-externalities/Cargo.toml | 4 +- utils/frame/remote-externalities/src/lib.rs | 1 + 3 files changed, 232 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 048782b5ebbb6..70fa4b9b60e96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,7 +350,7 @@ checksum = "2f23d769dbf1838d5df5156e7b1ad404f4c463d1ac2c6aeb6cd943630f8a8400" dependencies = [ "futures-core", "futures-io", - "rustls 0.19.0", + "rustls 0.19.1", "webpki 0.21.4", "webpki-roots", ] @@ -467,6 +467,9 @@ name = "beef" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6736e2428df2ca2848d846c43e88745121a6654696e349ce0054a420815a7409" +dependencies = [ + "serde", +] [[package]] name = "bincode" @@ -931,7 +934,17 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys 0.8.2", "libc", ] @@ -941,6 +954,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpp_demangle" version = "0.3.2" @@ -1593,7 +1612,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e43f2f1833d64e33f15592464d6fdd70f349dda7b1a53088eb83cd94014008c5" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", ] [[package]] @@ -1665,7 +1684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6447e2f8178843749e8c8003206def83ec124a7859475395777a28b5338647c" dependencies = [ "either", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "log", "num-traits", @@ -2008,9 +2027,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f55667319111d593ba876406af7c409c0ebb44dc4be6132a783ccf163ea14c1" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -2023,9 +2042,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2dd2df839b57db9ab69c2c9d8f3e8c81984781937fe2807dc6dcf3b2ad2939" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -2033,9 +2052,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15496a72fabf0e62bdc3df11a59a3787429221dd0710ba8ef163d6f7a9112c94" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-cpupool" @@ -2054,7 +2073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdcef58a173af8148b182684c9f2d5250875adbcaff7b5794073894f9d8634a9" dependencies = [ "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "lazy_static", "log", "parking_lot 0.9.0", @@ -2065,9 +2084,9 @@ dependencies = [ [[package]] name = "futures-executor" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891a4b7b96d84d5940084b2a37632dd65deeae662c114ceaa2c879629c9c0ad1" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -2077,9 +2096,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71c2c65c57704c32f5241c1223167c2c3294fd34ac020c807ddbe6db287ba59" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-lite" @@ -2098,10 +2117,11 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", "proc-macro2", "quote", @@ -2115,21 +2135,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a1387e07917c711fb4ee4f48ea0adb04a3c9739e53ef85bf43ae1edc2937a8b" dependencies = [ "futures-io", - "rustls 0.19.0", + "rustls 0.19.1", "webpki 0.21.4", ] [[package]] name = "futures-sink" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85754d98985841b7d4f5e8e6fbfa4a4ac847916893ec511a2917ccd8525b8bb3" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa189ef211c15ee602667a6fcfe1c1fd9e07d42250d2156382820fba33c9df80" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-timer" @@ -2149,10 +2169,11 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1812c7ab8aedf8d6f2701a43e1243acdbcc2b36ab26e2ad421eb99ac963d96d1" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures 0.1.31", "futures-channel", "futures-core", @@ -2496,6 +2517,17 @@ dependencies = [ "http 0.2.3", ] +[[package]] +name = "http-body" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +dependencies = [ + "bytes 1.0.1", + "http 0.2.3", + "pin-project-lite 0.2.6", +] + [[package]] name = "httparse" version = "1.3.5" @@ -2577,6 +2609,28 @@ dependencies = [ "want 0.3.0", ] +[[package]] +name = "hyper" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.3", + "http-body 0.4.2", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.5", + "tokio 1.6.0", + "tower-service", + "tracing", + "want 0.3.0", +] + [[package]] name = "hyper-rustls" version = "0.21.0" @@ -2589,7 +2643,7 @@ dependencies = [ "hyper 0.13.10", "log", "rustls 0.18.1", - "rustls-native-certs", + "rustls-native-certs 0.4.0", "tokio 0.2.25", "tokio-rustls", "webpki 0.21.4", @@ -2645,7 +2699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6d52908d4ea4ab2bc22474ba149bf1011c8e2c3ebc1ff593ae28ac44f494b6" dependencies = [ "async-io", - "futures 0.3.13", + "futures 0.3.15", "futures-lite", "if-addrs", "ipnet", @@ -2721,7 +2775,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64fa110ec7b8f493f416eed552740d10e7030ad5f63b2308f82c9608ec2df275" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "futures-timer 2.0.2", ] @@ -2925,9 +2979,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.2.0-alpha.6" +version = "0.2.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5784ee8bb31988fa2c7a755fe31b0e21aa51894a67e5c99b6d4470f0253bf31a" +checksum = "3e6075694416dfb53fd29910725a4de3dfee2370d4003827d9aafd10e060ef03" dependencies = [ "Inflector", "proc-macro-crate 1.0.0", @@ -2938,37 +2992,41 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.2.0-alpha.6" +version = "0.2.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab3dabceeeeb865897661d532d47202eaae71cd2c606f53cb69f1fbc0555a51" +checksum = "f60988e2759b2f1cf5b8366db430de14ec54a035659b6bcddefca50d306fe03f" dependencies = [ "async-trait", "beef", "futures-channel", "futures-util", + "hyper 0.14.5", "log", "serde", "serde_json", + "soketto 0.5.0", "thiserror", ] [[package]] name = "jsonrpsee-ws-client" -version = "0.2.0-alpha.6" +version = "0.2.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6fdb4390bd25358c62e8b778652a564a1723ba07dca0feb3da439c2253fe59f" +checksum = "7ac07a90b9fd6de3aedaf839a0d5b38bd481552b0927e567b234e1ed3c1349a4" dependencies = [ "async-std", "async-tls", "async-trait", "fnv", - "futures 0.3.13", + "futures 0.3.15", "jsonrpsee-types", "log", "pin-project 1.0.5", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", "serde", "serde_json", - "soketto", + "soketto 0.5.0", "thiserror", "url 2.2.1", "webpki 0.22.0", @@ -3055,7 +3113,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1e98ba343d0b35f9009a8844cd2b87fa3192f7e79033ac05b00aeae0f3b0b5" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "js-sys", "kvdb", "kvdb-memorydb", @@ -3125,7 +3183,7 @@ checksum = "08053fbef67cd777049ef7a95ebaca2ece370b4ed7712c3fa404d69a88cb741b" dependencies = [ "atomic", "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "lazy_static", "libp2p-core", "libp2p-deflate", @@ -3167,7 +3225,7 @@ dependencies = [ "ed25519-dalek", "either", "fnv", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "lazy_static", "libsecp256k1", @@ -3197,7 +3255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2181a641cd15f9b6ba71b1335800f309012a0a97a29ffaabbbf40e9d3d58f08" dependencies = [ "flate2", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", ] @@ -3208,7 +3266,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62e63dab8b5ff35e0c101a3e51e843ba782c07bbb1682f5fd827622e0d02b98b" dependencies = [ "async-std-resolver", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "log", "smallvec 1.6.1", @@ -3223,7 +3281,7 @@ checksum = "48a9b570f6766301d9c4aa00fce3554cad1598e2f466debbc4dde909028417cf" dependencies = [ "cuckoofilter", "fnv", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3244,7 +3302,7 @@ dependencies = [ "byteorder", "bytes 1.0.1", "fnv", - "futures 0.3.13", + "futures 0.3.15", "hex_fmt", "libp2p-core", "libp2p-swarm", @@ -3265,7 +3323,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f668f00efd9883e8b7bcc582eaf0164615792608f886f6577da18bcbeea0a46" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3286,7 +3344,7 @@ dependencies = [ "bytes 1.0.1", "either", "fnv", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3310,7 +3368,7 @@ dependencies = [ "async-io", "data-encoding", "dns-parser", - "futures 0.3.13", + "futures 0.3.15", "if-watch", "lazy_static", "libp2p-core", @@ -3330,7 +3388,7 @@ checksum = "85e9b544335d1ed30af71daa96edbefadef6f19c7a55f078b9fc92c87163105d" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "log", "nohash-hasher", @@ -3348,7 +3406,7 @@ checksum = "36db0f0db3b0433f5b9463f1c0cd9eadc0a3734a9170439ce501ff99733a88bd" dependencies = [ "bytes 1.0.1", "curve25519-dalek 3.0.2", - "futures 0.3.13", + "futures 0.3.15", "lazy_static", "libp2p-core", "log", @@ -3368,7 +3426,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4bfaffac63bf3c7ec11ed9d8879d455966ddea7e78ee14737f0b6dce0d1cd1" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3385,7 +3443,7 @@ checksum = "0c8c37b4d2a075b4be8442760a5f8c037180f0c8dd5b5734b9978ab868b3aa11" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "log", "prost", @@ -3400,7 +3458,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce3374f3b28162db9d3442c9347c4f14cb01e8290052615c7d341d40eae0599" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "log", "pin-project 1.0.5", "rand 0.7.3", @@ -3416,7 +3474,7 @@ checksum = "0b8786aca3f18671d8776289706a5521f6c9124a820f69e358de214b9939440d" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p-core", "libp2p-swarm", @@ -3439,7 +3497,7 @@ checksum = "1cdbe172f08e6d0f95fa8634e273d4c4268c4063de2e33e7435194b0130c62e3" dependencies = [ "async-trait", "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3458,7 +3516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e04d8e1eef675029ec728ba14e8d0da7975d84b6679b699b4ae91a1de9c3a92" dependencies = [ "either", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "log", "rand 0.7.3", @@ -3484,7 +3542,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b1a27d21c477951799e99d5c105d78868258502ce092988040a808d5a19bbd9" dependencies = [ "async-io", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "if-watch", "ipnet", @@ -3501,7 +3559,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd6564bb3b7ff203661ccbb69003c2b551e34cef974f2d6c6a28306a12170b5" dependencies = [ "async-std", - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "log", ] @@ -3512,7 +3570,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef45d61e43c313531b5e903e4e8415212ff6338e0c54c47da5b9b412b5760de" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "js-sys", "libp2p-core", "parity-send-wrapper", @@ -3527,13 +3585,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cace60995ef6f637e4752cccbb2590f6bc358e8741a0d066307636c69a4b3a74" dependencies = [ "either", - "futures 0.3.13", + "futures 0.3.15", "futures-rustls", "libp2p-core", "log", "quicksink", "rw-stream-sink", - "soketto", + "soketto 0.4.2", "url 2.2.1", "webpki-roots", ] @@ -3544,7 +3602,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f35da42cfc6d5cb0dcf3ad6881bc68d146cdf38f98655e09e33fbba4d13eabc4" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "libp2p-core", "parking_lot 0.11.1", "thiserror", @@ -3969,7 +4027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d91ec0a2440aaff5f78ec35631a7027d50386c6163aa975f7caa0d5da4b6ff8" dependencies = [ "bytes 1.0.1", - "futures 0.3.13", + "futures 0.3.15", "log", "pin-project 1.0.5", "smallvec 1.6.1", @@ -4042,7 +4100,7 @@ version = "0.8.0" dependencies = [ "derive_more", "fs_extra", - "futures 0.3.13", + "futures 0.3.15", "hash-db", "hex", "kvdb", @@ -4078,7 +4136,7 @@ dependencies = [ name = "node-browser-testing" version = "2.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "jsonrpc-core", "libp2p", @@ -4100,7 +4158,7 @@ dependencies = [ "frame-benchmarking-cli", "frame-support", "frame-system", - "futures 0.3.13", + "futures 0.3.15", "hex-literal", "libp2p-wasm-ext", "log", @@ -4148,7 +4206,7 @@ dependencies = [ "sc-transaction-pool", "serde", "serde_json", - "soketto", + "soketto 0.4.2", "sp-authority-discovery", "sp-authorship", "sp-consensus", @@ -4181,7 +4239,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "futures 0.3.13", + "futures 0.3.15", "node-primitives", "node-runtime", "node-testing", @@ -4440,7 +4498,7 @@ dependencies = [ "frame-support", "frame-system", "fs_extra", - "futures 0.3.13", + "futures 0.3.15", "log", "node-executor", "node-primitives", @@ -6791,9 +6849,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", "log", @@ -6811,7 +6869,19 @@ dependencies = [ "openssl-probe", "rustls 0.18.1", "schannel", - "security-framework", + "security-framework 1.0.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework 2.2.0", ] [[package]] @@ -6836,7 +6906,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "pin-project 0.4.27", "static_assertions", ] @@ -6881,7 +6951,7 @@ dependencies = [ "async-trait", "derive_more", "either", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "ip_network", "libp2p", @@ -6910,7 +6980,7 @@ dependencies = [ name = "sc-basic-authorship" version = "0.9.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -6984,7 +7054,7 @@ version = "0.9.0" dependencies = [ "chrono", "fdlimit", - "futures 0.3.13", + "futures 0.3.15", "hex", "libp2p", "log", @@ -7022,7 +7092,7 @@ version = "3.0.0" dependencies = [ "derive_more", "fnv", - "futures 0.3.13", + "futures 0.3.15", "hash-db", "kvdb", "kvdb-memorydb", @@ -7104,7 +7174,7 @@ version = "0.9.0" dependencies = [ "async-trait", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "getrandom 0.2.2", "log", @@ -7147,7 +7217,7 @@ dependencies = [ "async-trait", "derive_more", "fork-tree", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "log", "merlin", @@ -7201,7 +7271,7 @@ name = "sc-consensus-babe-rpc" version = "0.9.0" dependencies = [ "derive_more", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -7244,7 +7314,7 @@ dependencies = [ "assert_matches", "async-trait", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -7282,7 +7352,7 @@ version = "0.9.0" dependencies = [ "async-trait", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -7304,7 +7374,7 @@ name = "sc-consensus-slots" version = "0.9.0" dependencies = [ "async-trait", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "impl-trait-for-tuples", "log", @@ -7436,7 +7506,7 @@ dependencies = [ "dyn-clone", "finality-grandpa", "fork-tree", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "linked-hash-map", "log", @@ -7481,7 +7551,7 @@ version = "0.9.0" dependencies = [ "derive_more", "finality-grandpa", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -7511,7 +7581,7 @@ version = "0.9.0" dependencies = [ "derive_more", "finality-grandpa", - "futures 0.3.13", + "futures 0.3.15", "log", "num-traits", "parity-scale-codec", @@ -7536,7 +7606,7 @@ name = "sc-informant" version = "0.9.0" dependencies = [ "ansi_term 0.12.1", - "futures 0.3.13", + "futures 0.3.15", "log", "parity-util-mem", "sc-client-api", @@ -7554,7 +7624,7 @@ version = "3.0.0" dependencies = [ "async-trait", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "futures-util", "hex", "merlin", @@ -7603,7 +7673,7 @@ dependencies = [ "erased-serde", "fnv", "fork-tree", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "hex", "ip_network", @@ -7651,7 +7721,7 @@ name = "sc-network-gossip" version = "0.9.0" dependencies = [ "async-std", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p", "log", @@ -7672,7 +7742,7 @@ version = "0.8.0" dependencies = [ "async-std", "async-trait", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p", "log", @@ -7700,7 +7770,7 @@ version = "3.0.0" dependencies = [ "bytes 0.5.6", "fnv", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "hex", "hyper 0.13.10", @@ -7734,7 +7804,7 @@ dependencies = [ name = "sc-peerset" version = "3.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "libp2p", "log", "rand 0.7.3", @@ -7757,7 +7827,7 @@ version = "3.0.0" dependencies = [ "assert_matches", "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "hash-db", "jsonrpc-core", "jsonrpc-pubsub", @@ -7799,7 +7869,7 @@ name = "sc-rpc-api" version = "0.9.0" dependencies = [ "derive_more", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -7858,7 +7928,7 @@ dependencies = [ "directories", "exit-future", "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "hash-db", "jsonrpc-core", @@ -7925,7 +7995,7 @@ version = "2.0.0" dependencies = [ "fdlimit", "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "hex-literal", "log", "parity-scale-codec", @@ -7993,7 +8063,7 @@ name = "sc-telemetry" version = "3.0.0" dependencies = [ "chrono", - "futures 0.3.13", + "futures 0.3.15", "libp2p", "log", "parking_lot 0.11.1", @@ -8060,7 +8130,7 @@ dependencies = [ "assert_matches", "criterion", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "linked-hash-map", "log", "parity-scale-codec", @@ -8083,7 +8153,7 @@ name = "sc-transaction-pool" version = "3.0.0" dependencies = [ "assert_matches", - "futures 0.3.13", + "futures 0.3.15", "futures-diagnose", "hex", "intervalier", @@ -8197,10 +8267,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad502866817f0575705bd7be36e2b2535cc33262d493aa733a2ec862baa2bc2b" dependencies = [ "bitflags", - "core-foundation", - "core-foundation-sys", + "core-foundation 0.7.0", + "core-foundation-sys 0.7.0", + "libc", + "security-framework-sys 1.0.0", +] + +[[package]] +name = "security-framework" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +dependencies = [ + "bitflags", + "core-foundation 0.9.1", + "core-foundation-sys 0.8.2", "libc", - "security-framework-sys", + "security-framework-sys 2.2.0", ] [[package]] @@ -8209,7 +8292,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ceb04988b17b6d1dcd555390fa822ca5637b4a14e1f5099f13d351bed4d6c7" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys 0.8.2", "libc", ] @@ -8501,13 +8594,28 @@ dependencies = [ "base64 0.12.3", "bytes 0.5.6", "flate2", - "futures 0.3.13", + "futures 0.3.15", "httparse", "log", "rand 0.7.3", "sha-1 0.9.4", ] +[[package]] +name = "soketto" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4919971d141dbadaa0e82b5d369e2d7666c98e4625046140615ca363e50d4daa" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "futures 0.3.15", + "httparse", + "log", + "rand 0.8.3", + "sha-1 0.9.4", +] + [[package]] name = "sp-allocator" version = "3.0.0" @@ -8656,7 +8764,7 @@ dependencies = [ name = "sp-blockchain" version = "3.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "log", "lru", "parity-scale-codec", @@ -8682,7 +8790,7 @@ name = "sp-consensus" version = "0.9.0" dependencies = [ "async-trait", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p", "log", @@ -8782,7 +8890,7 @@ dependencies = [ "criterion", "dyn-clonable", "ed25519-dalek", - "futures 0.3.13", + "futures 0.3.15", "hash-db", "hash256-std-hasher", "hex", @@ -8869,7 +8977,7 @@ name = "sp-inherents" version = "3.0.0" dependencies = [ "async-trait", - "futures 0.3.13", + "futures 0.3.15", "impl-trait-for-tuples", "parity-scale-codec", "sp-core", @@ -8882,7 +8990,7 @@ dependencies = [ name = "sp-io" version = "3.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "hash-db", "libsecp256k1", "log", @@ -8918,7 +9026,7 @@ version = "0.9.0" dependencies = [ "async-trait", "derive_more", - "futures 0.3.13", + "futures 0.3.15", "merlin", "parity-scale-codec", "parking_lot 0.11.1", @@ -9253,7 +9361,7 @@ name = "sp-transaction-pool" version = "3.0.0" dependencies = [ "derive_more", - "futures 0.3.13", + "futures 0.3.15", "log", "parity-scale-codec", "serde", @@ -9285,7 +9393,7 @@ dependencies = [ name = "sp-utils" version = "3.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "futures-core", "futures-timer 3.0.2", "lazy_static", @@ -9451,7 +9559,7 @@ dependencies = [ "chrono", "console_error_panic_hook", "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "futures-timer 3.0.2", "getrandom 0.2.2", "js-sys", @@ -9494,7 +9602,7 @@ version = "3.0.0" dependencies = [ "frame-support", "frame-system", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-client-transports", "jsonrpc-core", "parity-scale-codec", @@ -9509,7 +9617,7 @@ name = "substrate-frame-rpc-system" version = "3.0.0" dependencies = [ "frame-system-rpc-runtime-api", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -9548,7 +9656,7 @@ version = "2.0.1" dependencies = [ "async-trait", "futures 0.1.31", - "futures 0.3.13", + "futures 0.3.15", "hash-db", "hex", "parity-scale-codec", @@ -9578,7 +9686,7 @@ dependencies = [ "frame-support", "frame-system", "frame-system-rpc-runtime-api", - "futures 0.3.13", + "futures 0.3.15", "log", "memory-db", "pallet-babe", @@ -9618,7 +9726,7 @@ dependencies = [ name = "substrate-test-runtime-client" version = "2.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "parity-scale-codec", "sc-block-builder", "sc-client-api", @@ -9639,7 +9747,7 @@ name = "substrate-test-runtime-transaction-pool" version = "2.0.0" dependencies = [ "derive_more", - "futures 0.3.13", + "futures 0.3.15", "parity-scale-codec", "parking_lot 0.11.1", "sc-transaction-graph", @@ -9653,7 +9761,7 @@ dependencies = [ name = "substrate-test-utils" version = "3.0.0" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "sc-service", "substrate-test-utils-derive", "tokio 0.2.25", @@ -9775,7 +9883,7 @@ version = "0.9.0" dependencies = [ "env_logger 0.7.1", "frame-system", - "futures 0.3.13", + "futures 0.3.15", "jsonrpc-core", "log", "sc-basic-authorship", @@ -10848,7 +10956,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "js-sys", "parking_lot 0.11.1", "pin-utils", @@ -11264,7 +11372,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ - "futures 0.3.13", + "futures 0.3.15", "log", "nohash-hasher", "parking_lot 0.11.1", diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 153648ddc8be1..bf5b3f50b083b 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -13,8 +13,8 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee-ws-client = { version = "=0.2.0-alpha.6", default-features = false } -jsonrpsee-proc-macros = "=0.2.0-alpha.6" +jsonrpsee-ws-client = { version = "=0.2.0-alpha.7", default-features = false } +jsonrpsee-proc-macros = "=0.2.0-alpha.7" serde_json = "1.0" serde = "1.0.0" diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index f09348dcc9ec9..45f83c413385e 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -439,6 +439,7 @@ impl Builder { let ws_client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(&online.transport.uri) + .await .map_err(|e| "failed to build ws client")?; online.transport.client = Some(ws_client); From 13f9200467a5296552ab82282de260facd98df05 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 27 May 2021 20:44:47 -0700 Subject: [PATCH 13/52] WIP --- utils/frame/try-runtime/cli/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index c3ed1af5ab31d..f168e4b0f5c3b 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -79,7 +79,7 @@ pub struct SharedParams { /// Various commands to try out against runtime state at a specific block. /// -/// NOTE: For now this only assumes running the `on_runtime_upgrade` hooks. // TOOD wut dis mean? +/// NOTE: For now this only assumes running the `on_runtime_upgrade` hooks #[derive(Debug, Clone, structopt::StructOpt)] pub struct TryRuntimeCmd { #[structopt(flatten)] @@ -98,6 +98,10 @@ pub enum State { }, /// Use a live chain as the source of runtime state. + /// + /// NOTE: There currently is a bug when connecting via WS to localhost. To get around run the + /// node with `--rpc-cors=all` + /// https://github.com/paritytech/jsonrpsee/issues/346 Live { /// An optional state snapshot file to WRITE to. #[structopt(short, long)] @@ -275,7 +279,7 @@ where sp_core::testing::TaskExecutor::new(), ) .execute(execution.into()) - .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; + .map_err(|e| format!("failed to execute 'OffchainWorkerApi_offchain_worker' due to {:?}", e))?; log::info!("OffchainWorkerApi_offchain_worker executed without errors."); Ok(()) From ea13bff5b88c3aef75d1199abb89ecdb53b25f49 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 31 May 2021 16:59:40 -0700 Subject: [PATCH 14/52] Executiing without errors --- Cargo.lock | 3 +++ utils/frame/remote-externalities/Cargo.toml | 2 -- utils/frame/remote-externalities/src/lib.rs | 22 +++++++++++++++------ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7efc6e04f3ade..7da8cff598964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "Inflector" version = "0.11.4" @@ -7621,6 +7623,7 @@ version = "0.9.0" dependencies = [ "ansi_term 0.12.1", "futures 0.3.15", + "futures-timer 3.0.2", "log", "parity-util-mem", "sc-client-api", diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index ec8c2fc5a798e..d4ddd6362fc62 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -24,8 +24,6 @@ env_logger = "0.8.2" log = "0.4.11" codec = { package = "parity-scale-codec", version = "2.0.0" } -serde_json = "1.0" - sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-core = { version = "3.0.0", path = "../../../primitives/core" } sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 403701793c913..d77e81c24b6b2 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -104,7 +104,6 @@ use std::{ fs, path::{Path, PathBuf}, - error, }; use log::*; use sp_core::hashing::twox_128; @@ -112,11 +111,12 @@ pub use sp_io::TestExternalities; use sp_core::{ hexdisplay::HexDisplay, storage::{StorageKey, StorageData}, + offchain::{OffchainWorkerExt, OffchainDbExt, testing::TestOffchainExt}, }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; use jsonrpsee_ws_client::{ - WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client, + WsClientBuilder, WsClient, v2::params::JsonRpcParams, }; type KeyPair = (StorageKey, StorageData); @@ -154,7 +154,8 @@ impl Default for Mode { } } -/// configuration of the online execution. +// TODO are there instructions for creating a snapshot +/// Configuration of the offline execution. /// /// A state snapshot config must be present. #[derive(Clone)] @@ -163,7 +164,7 @@ pub struct OfflineConfig { pub state_snapshot: SnapshotConfig, } -/// Description of the transport protocol. +/// Description of the transport protocol (for online execution). #[derive(Debug)] pub struct Transport { uri: String, @@ -209,7 +210,9 @@ impl Default for OnlineConfig { } // TODO -pub async fn get_header>(from: S, at: B::Hash) -> B::Header where B::Header: serde::de::DeserializeOwned { +pub async fn get_header>(from: S, at: B::Hash) -> B::Header + where B::Header: serde::de::DeserializeOwned +{ use jsonrpsee_ws_client::traits::Client; let params = vec![serde_json::to_value(at).unwrap()]; let client = WsClientBuilder::default() @@ -353,6 +356,7 @@ impl Builder { prefix: StorageKey, at: B::Hash, ) -> Result, &'static str> { + use jsonrpsee_ws_client::traits::Client; use serde_json::to_value; let keys = self.get_keys_paged(prefix, at).await?; let keys_count = keys.len(); @@ -465,7 +469,7 @@ impl Builder { .max_request_body_size(u32::MAX) .build(&online.transport.uri) .await - .map_err(|e| "failed to build ws client")?; + .map_err(|_| "failed to build ws client")?; online.transport.client = Some(ws_client); // Then, if `at` is not set, set it. @@ -525,12 +529,18 @@ impl Builder { pub async fn build(self) -> Result { let kv = self.pre_build().await?; let mut ext = TestExternalities::new_empty(); + let (offchain, _offchain_state) = TestOffchainExt::new(); + + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); for (k, v) in kv { let (k, v) = (k.0, v.0); + // Insert the key,value pair into the test trie backend ext.insert(k, v); } + Ok(ext) } } From a12682899c430b1f6c3031ab8a0813b210e3d2b6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 31 May 2021 17:35:18 -0700 Subject: [PATCH 15/52] Reorg & cleanup --- utils/frame/remote-externalities/Cargo.toml | 5 +- utils/frame/remote-externalities/src/lib.rs | 69 ++++++++++++--------- utils/frame/try-runtime/cli/src/lib.rs | 24 ++++--- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index d4ddd6362fc62..7ed7ce3a3c6fc 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -16,13 +16,12 @@ targets = ["x86_64-unknown-linux-gnu"] jsonrpsee-ws-client = { version = "=0.2.0-alpha.7", default-features = false } jsonrpsee-proc-macros = "=0.2.0-alpha.7" -serde_json = "1.0" -serde = "1.0.0" - hex = "0.4.0" env_logger = "0.8.2" log = "0.4.11" codec = { package = "parity-scale-codec", version = "2.0.0" } +serde_json = "1.0" +serde = "1.0.0" sp-io = { version = "3.0.0", path = "../../../primitives/io" } sp-core = { version = "3.0.0", path = "../../../primitives/core" } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index d77e81c24b6b2..88d42b267ba67 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -154,7 +154,6 @@ impl Default for Mode { } } -// TODO are there instructions for creating a snapshot /// Configuration of the offline execution. /// /// A state snapshot config must be present. @@ -198,6 +197,13 @@ pub struct OnlineConfig { pub transport: Transport, } +impl OnlineConfig { + /// Return rpc (ws) client. + fn rpc_client(&self) -> &WsClient { + self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.") + } +} + impl Default for OnlineConfig { fn default() -> Self { Self { @@ -209,36 +215,6 @@ impl Default for OnlineConfig { } } -// TODO -pub async fn get_header>(from: S, at: B::Hash) -> B::Header - where B::Header: serde::de::DeserializeOwned -{ - use jsonrpsee_ws_client::traits::Client; - let params = vec![serde_json::to_value(at).unwrap()]; - let client = WsClientBuilder::default() - .max_request_body_size(u32::MAX) - .build(from.as_ref()) - .await - .unwrap(); - client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() -} - -pub async fn get_finalized_head>(from: S) -> B::Hash { - use jsonrpsee_ws_client::traits::Client; - let client = WsClientBuilder::default() - .max_request_body_size(u32::MAX) - .build(from.as_ref()) - .await - .unwrap(); - client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() -} - -impl OnlineConfig { - /// Return rpc (ws) client. - fn rpc_client(&self) -> &WsClient { - self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.") - } -} /// Configuration of the state snapshot. #[derive(Clone)] @@ -531,6 +507,7 @@ impl Builder { let mut ext = TestExternalities::new_empty(); let (offchain, _offchain_state) = TestOffchainExt::new(); + // Register externality extensions in order to provide host interface for OCW to the runtime ext.register_extension(OffchainDbExt::new(offchain.clone())); ext.register_extension(OffchainWorkerExt::new(offchain)); @@ -545,6 +522,36 @@ impl Builder { } } +/// WS RPC API for one RPC calls to a substrate node. +pub mod rpc_api { + use super::*; + /// Get the header of the block identified by `at` + pub async fn get_header>(from: S, at: B::Hash) -> B::Header + where + B::Header: serde::de::DeserializeOwned, + { + use jsonrpsee_ws_client::traits::Client; + let params = vec![serde_json::to_value(at).unwrap()]; + let client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(from.as_ref()) + .await + .unwrap(); + client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() + } + + /// Get the finalized head + pub async fn get_finalized_head>(from: S) -> B::Hash { + use jsonrpsee_ws_client::traits::Client; + let client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(from.as_ref()) + .await + .unwrap(); + client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() + } +} + #[cfg(test)] mod test_prelude { pub(crate) use super::*; diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index f168e4b0f5c3b..eb7d9fc456b40 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -26,7 +26,7 @@ use sc_service::NativeExecutionDispatch; use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::storage::{StorageData, StorageKey, well_known_keys}; -use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig}; +use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig, rpc_api}; #[derive(Debug, Clone, structopt::StructOpt)] pub enum Command { @@ -42,7 +42,7 @@ pub struct OnRuntimeUpgradeCmd { #[derive(Debug, Clone, structopt::StructOpt)] pub struct OffchainWorkerCmd { - #[structopt(short, long, multiple = false, parse(try_from_str = util::parse_hash))] + #[structopt(short, long, multiple = false, parse(try_from_str = parse::hash))] pub header_at: String, #[structopt(subcommand)] @@ -109,7 +109,7 @@ pub enum State { /// The block hash at which to connect. /// Will be latest finalized head if not provided. - #[structopt(short, long, multiple = false, parse(try_from_str = util::parse_hash))] + #[structopt(short, long, multiple = false, parse(try_from_str = parse::hash))] block_at: Option, /// The modules to scrape. If empty, entire chain state will be scraped. @@ -117,7 +117,7 @@ pub enum State { modules: Option>, /// The url to connect to. - #[structopt(default_value = "ws://localhost:9944", parse(try_from_str = util::parse_url))] + #[structopt(default_value = "ws://localhost:9944", parse(try_from_str = parse::url))] url: String, }, } @@ -209,7 +209,11 @@ where Ok(()) } -async fn offchain_worker(shared: SharedParams, command: OffchainWorkerCmd, config: Configuration) -> sc_cli::Result<()> +async fn offchain_worker( + shared: SharedParams, + command: OffchainWorkerCmd, + config: Configuration, +)-> sc_cli::Result<()> where B: BlockT, B::Hash: FromStr, @@ -261,7 +265,7 @@ where }; let header_hash: B::Hash = command.header_at.parse().unwrap(); - let header = remote_externalities::get_header::( + let header = rpc_api::get_header::( url, header_hash ).await; @@ -320,8 +324,9 @@ impl CliConfiguration for TryRuntimeCmd { } } -mod util { - pub fn parse_hash(block_number: &str) -> Result { +// Utils for parsing user input +mod parse { + pub fn hash(block_number: &str) -> Result { let block_number = if block_number.starts_with("0x") { &block_number[2..] } else { @@ -338,7 +343,7 @@ mod util { } } - pub fn parse_url(s: &str) -> Result { + pub fn url(s: &str) -> Result { if s.starts_with("ws://") || s.starts_with("wss://") { // could use Url crate as well, but lets keep it simple for now. Ok(s.to_string()) @@ -347,3 +352,4 @@ mod util { } } } + From 2746f6947ceaee5db28d67e33b6d9b5946a3e22e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 31 May 2021 18:24:50 -0700 Subject: [PATCH 16/52] Trivial cleaning --- utils/frame/try-runtime/cli/src/lib.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index eb7d9fc456b40..8bd6f96fe52cf 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -259,16 +259,14 @@ where }; let ext = { - let builder = Builder::::new().mode(Mode::Online(online_config)); - - builder.build().await? + Builder::::new() + .mode(Mode::Online(online_config)) + .build() + .await? }; let header_hash: B::Hash = command.header_at.parse().unwrap(); - let header = rpc_api::get_header::( - url, - header_hash - ).await; + let header = rpc_api::get_header::(url,header_hash).await; let _ = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, @@ -286,6 +284,7 @@ where .map_err(|e| format!("failed to execute 'OffchainWorkerApi_offchain_worker' due to {:?}", e))?; log::info!("OffchainWorkerApi_offchain_worker executed without errors."); + Ok(()) } From 0961bd4004f709e9428bea7a4baf49eace66f978 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:32:22 -0700 Subject: [PATCH 17/52] Add txpool & keystore extension --- Cargo.lock | 1 + bin/node/executor/tests/submit_transaction.rs | 2 +- utils/frame/remote-externalities/src/lib.rs | 14 ++-- utils/frame/try-runtime/cli/Cargo.toml | 1 + utils/frame/try-runtime/cli/src/lib.rs | 64 +++++++++++++------ 5 files changed, 54 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7da8cff598964..9596bd2e53625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10599,6 +10599,7 @@ dependencies = [ "sp-blockchain", "sp-core", "sp-externalities", + "sp-keystore", "sp-runtime", "sp-state-machine", "structopt", diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 3de0758d81462..09c74651ea286 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -26,7 +26,7 @@ use sp_core::{ testing::TestTransactionPoolExt, }, }; -use sp_keystore::{KeystoreExt, SyncCryptoStore, testing::KeyStore}; +use sp_keystore::{KeystoreExt, testing::KeyStore}; use frame_system::{ offchain::{ Signer, diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 88d42b267ba67..d5b1c2f077426 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -111,7 +111,7 @@ pub use sp_io::TestExternalities; use sp_core::{ hexdisplay::HexDisplay, storage::{StorageKey, StorageData}, - offchain::{OffchainWorkerExt, OffchainDbExt, testing::TestOffchainExt}, + // offchain::{OffchainWorkerExt, OffchainDbExt, testing::TestOffchainExt}, }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; @@ -247,7 +247,10 @@ pub struct Builder { // that. impl Default for Builder { fn default() -> Self { - Self { inject: Default::default(), mode: Default::default() } + Self { + inject: Default::default(), + mode: Default::default(), + } } } @@ -505,11 +508,6 @@ impl Builder { pub async fn build(self) -> Result { let kv = self.pre_build().await?; let mut ext = TestExternalities::new_empty(); - let (offchain, _offchain_state) = TestOffchainExt::new(); - - // Register externality extensions in order to provide host interface for OCW to the runtime - ext.register_extension(OffchainDbExt::new(offchain.clone())); - ext.register_extension(OffchainWorkerExt::new(offchain)); info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); for (k, v) in kv { @@ -522,7 +520,7 @@ impl Builder { } } -/// WS RPC API for one RPC calls to a substrate node. +/// WS RPC API for one off RPC calls to a substrate node. pub mod rpc_api { use super::*; /// Get the header of the block identified by `at` diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index ecb43ebd8f114..86a87a0a9e93a 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -28,6 +28,7 @@ sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" sp-runtime = { version = "3.0.0", path = "../../../../primitives/runtime" } sp-externalities = { version = "0.9.0", path = "../../../../primitives/externalities" } sp-core = { version = "3.0.0", path = "../../../../primitives/core" } +sp-keystore = { version = "0.9.0", path = "../../../../primitives/keystore" } frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" } remote-externalities = { version = "0.9.0", path = "../../remote-externalities" } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 8bd6f96fe52cf..2ae97ffe5defd 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -18,14 +18,21 @@ //! `Structopt`-ready structs for `try-runtime`. use parity_scale_codec::{Decode, Encode}; -use std::{fmt::Debug, path::PathBuf, str::FromStr}; +use std::{fmt::Debug, path::PathBuf, str::FromStr, sync::Arc}; use sc_service::Configuration; use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; use sc_executor::NativeExecutor; use sc_service::NativeExecutionDispatch; use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; -use sp_core::storage::{StorageData, StorageKey, well_known_keys}; +use sp_core::{ + offchain::{ + OffchainWorkerExt, OffchainDbExt, TransactionPoolExt, + testing::{TestOffchainExt, TestTransactionPoolExt} + }, + storage::{StorageData, StorageKey, well_known_keys}, +}; +use sp_keystore::{KeystoreExt, testing::KeyStore}; use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig, rpc_api}; #[derive(Debug, Clone, structopt::StructOpt)] @@ -92,16 +99,13 @@ pub struct TryRuntimeCmd { /// The source of runtime state to try operations against. #[derive(Debug, Clone, structopt::StructOpt)] pub enum State { - /// Use a state snapshot as the source of runtime state. + /// Use a state snapshot as the source of runtime state. NOTE: at the moment this is only + /// paritially supported for offchain-worker. Snap { snapshot_path: PathBuf, }, /// Use a live chain as the source of runtime state. - /// - /// NOTE: There currently is a bug when connecting via WS to localhost. To get around run the - /// node with `--rpc-cors=all` - /// https://github.com/paritytech/jsonrpsee/issues/346 Live { /// An optional state snapshot file to WRITE to. #[structopt(short, long)] @@ -122,7 +126,11 @@ pub enum State { }, } -async fn on_runtime_upgrade(shared: SharedParams, command: OnRuntimeUpgradeCmd, config: Configuration) -> sc_cli::Result<()> +async fn on_runtime_upgrade( + shared: SharedParams, + command: OnRuntimeUpgradeCmd, + config: Configuration +) -> sc_cli::Result<()> where B: BlockT, B::Hash: FromStr, @@ -235,7 +243,7 @@ where max_runtime_instances, ); - let (online_config, url) = match command.state { + let (mode, url) = match command.state { State::Live { url, snapshot_path, @@ -251,19 +259,37 @@ where ..Default::default() }; - (online_config, url) + (Mode::Online(online_config), url) }, - _ => { - panic!("Only live state is supported"); - } - }; + State::Snap { snapshot_path } => { + // This is a temporary hack; the url is used just to get the header. We should try + // and get the header out of state, OR use an arbitrary header if thats ok, OR allow + // the user to feed in a header via file. + // This assumes you have a node running on local host default + let url = "ws://127.0.0.1:9944".to_string(); + let mode = Mode::Offline(OfflineConfig { + state_snapshot: SnapshotConfig::new(snapshot_path), + }); - let ext = { - Builder::::new() - .mode(Mode::Online(online_config)) - .build() - .await? + (mode, url) + } }; + let mut ext = Builder::::new() + .mode(mode) + .build() + .await?; + // Register externality extensions in order to provide host interface for OCW to the runtime + let (offchain, _offchain_state) = TestOffchainExt::new(); + let (pool, _pool_state) = TestTransactionPoolExt::new(); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); + let keystore_ptr = Arc::new(KeyStore::new()); + ext.register_extension(KeystoreExt(keystore_ptr)); + ext.register_extension(TransactionPoolExt::new(pool)); + + + // inject the code into this ext. + // builder.inject(&[(code_key, code)]).build().await?; let header_hash: B::Hash = command.header_at.parse().unwrap(); let header = rpc_api::get_header::(url,header_hash).await; From 99fdf47e6a649a7a9b0ee3ea62d38f76ac05c26c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 1 Jun 2021 18:24:38 -0700 Subject: [PATCH 18/52] Small cleaning --- utils/frame/try-runtime/cli/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index afd7d35813541..661bce9f3614c 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -105,7 +105,7 @@ pub enum State { /// Use a live chain as the source of runtime state. Live { - /// An optional state snapshot file to WRITE to. + /// An optional state snapshot file to WRITE to. Not written if set to `None`. #[structopt(short, long)] snapshot_path: Option, @@ -249,12 +249,12 @@ where modules } => { let online_config = OnlineConfig { - transport: url.to_owned().into(), - state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - modules: modules.to_owned().unwrap_or_default(), - at: block_at.as_ref() - .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, - ..Default::default() + transport: url.to_owned().into(), + state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), + modules: modules.to_owned().unwrap_or_default(), + at: block_at.as_ref() + .map(|b| b.parse().map_err(|e| format!("Could not parse hash: {:?}", e))).transpose()?, + ..Default::default() }; (Mode::Online(online_config), url) @@ -290,13 +290,13 @@ where .inject(&[(code_key, code)]) .build() .await?; - // Register externality extensions in order to provide host interface for OCW to the runtime + + // register externality extensions in order to provide host interface for OCW to the runtime let (offchain, _offchain_state) = TestOffchainExt::new(); let (pool, _pool_state) = TestTransactionPoolExt::new(); ext.register_extension(OffchainDbExt::new(offchain.clone())); ext.register_extension(OffchainWorkerExt::new(offchain)); - let keystore_ptr = Arc::new(KeyStore::new()); - ext.register_extension(KeystoreExt(keystore_ptr)); + ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()))); ext.register_extension(TransactionPoolExt::new(pool)); let header_hash: B::Hash = command.header_at.parse().unwrap(); From 6b5ae7f1ec579a31c0fda84aff2d8db7bd91baea Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 1 Jun 2021 18:55:01 -0700 Subject: [PATCH 19/52] More :cleaning --- bin/node/executor/tests/submit_transaction.rs | 2 +- utils/frame/remote-externalities/src/lib.rs | 1 + utils/frame/try-runtime/cli/src/lib.rs | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 09c74651ea286..3de0758d81462 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -26,7 +26,7 @@ use sp_core::{ testing::TestTransactionPoolExt, }, }; -use sp_keystore::{KeystoreExt, testing::KeyStore}; +use sp_keystore::{KeystoreExt, SyncCryptoStore, testing::KeyStore}; use frame_system::{ offchain::{ Signer, diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 68602cefc2c8a..cedda630eaf97 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -451,6 +451,7 @@ impl Builder { } /// WS RPC API for one off RPC calls to a substrate node. +/// TODO: Consolidate one off RPC calls https://github.com/paritytech/substrate/issues/8988 pub mod rpc_api { use super::*; /// Get the header of the block identified by `at` diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 661bce9f3614c..7caadfa55e36e 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -97,8 +97,9 @@ pub struct TryRuntimeCmd { /// The source of runtime state to try operations against. #[derive(Debug, Clone, structopt::StructOpt)] pub enum State { - /// Use a state snapshot as the source of runtime state. NOTE: at the moment this is only - /// paritially supported for offchain-worker. + /// Use a state snapshot as the source of runtime state. NOTE: for the offchain-worker command this + /// is only partially supported at the moment and you must have a relevant archive node exposed on + //// localhost:9944 in order to query the block header. Snap { snapshot_path: PathBuf, }, From 6e3746e28c1e9056866eb63e96486dcc40a7a5d5 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 3 Jun 2021 17:40:05 -0700 Subject: [PATCH 20/52] Flags: page-size, override-code --- utils/frame/try-runtime/cli/src/lib.rs | 58 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 7caadfa55e36e..852c0b7d57516 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -49,11 +49,22 @@ pub struct OnRuntimeUpgradeCmd { #[derive(Debug, Clone, structopt::StructOpt)] pub struct OffchainWorkerCmd { + /// Hash of the block whose header to use to execute the offchain worker. #[structopt(short, long, multiple = false, parse(try_from_str = parse::hash))] pub header_at: String, #[structopt(subcommand)] pub state: State, + + /// Wether or not to overwrite the code from state with the code from + /// the specified chain spec. + #[structopt(long)] + pub overwrite_code: bool, + + /// The number of 64KB pages to allocate for Wasm execution. Defaults to + /// sc_service::Configuration.default_heap_pages. + #[structopt(long, short)] + pub heap_pages: Option, } #[derive(Debug, Clone, structopt::StructOpt)] @@ -152,11 +163,10 @@ where let wasm_method = shared.wasm_method; let execution = shared.execution; - let mut changes = Default::default(); - // don't really care about these -- use the default values. - let max_runtime_instances = config.max_runtime_instances; + // don't really care about these so we just use the default let heap_pages = config.default_heap_pages; + let max_runtime_instances = config.max_runtime_instances; let executor = NativeExecutor::::new( wasm_method.into(), heap_pages, @@ -232,10 +242,13 @@ where { let wasm_method = shared.wasm_method; let execution = shared.execution; - + let heap_pages = if command.heap_pages.is_some() { + command.heap_pages + } else { + config.default_heap_pages + }; let mut changes = Default::default(); let max_runtime_instances = config.max_runtime_instances; - let heap_pages = config.default_heap_pages; let executor = NativeExecutor::::new( wasm_method.into(), heap_pages, @@ -273,24 +286,23 @@ where (mode, url) } }; - - // get the code to inject inject into the ext - let spec = config.chain_spec; - let genesis_storage = spec.build_storage()?; - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); - let code_key = StorageKey(well_known_keys::CODE.to_vec()); - - let mut ext = Builder::::new() - .mode(mode) - .inject(&[(code_key, code)]) - .build() - .await?; + let builder = Builder::::new().mode(mode); + let mut ext = if command.overwrite_code { + // get the code to inject inject into the ext + let spec = config.chain_spec; // TODO DRY this code block + let genesis_storage = spec.build_storage()?; + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + builder.inject(&[(code_key, code)]).build().await? + } else { + builder.build().await? + }; // register externality extensions in order to provide host interface for OCW to the runtime let (offchain, _offchain_state) = TestOffchainExt::new(); From 6e352c42f34b72410d93e1f8b1eb25737b12e339 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 09:31:16 -0700 Subject: [PATCH 21/52] WIP --- utils/frame/try-runtime/cli/src/lib.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 852c0b7d57516..621176c8c8843 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -60,11 +60,6 @@ pub struct OffchainWorkerCmd { /// the specified chain spec. #[structopt(long)] pub overwrite_code: bool, - - /// The number of 64KB pages to allocate for Wasm execution. Defaults to - /// sc_service::Configuration.default_heap_pages. - #[structopt(long, short)] - pub heap_pages: Option, } #[derive(Debug, Clone, structopt::StructOpt)] @@ -93,6 +88,11 @@ pub struct SharedParams { default_value = "compiled" )] pub wasm_method: WasmExecutionMethod, + + /// The number of 64KB pages to allocate for Wasm execution. Defaults to + /// sc_service::Configuration.default_heap_pages. + #[structopt(long)] + pub heap_pages: Option, } /// Various commands to try out against runtime state at a specific block. @@ -163,9 +163,13 @@ where let wasm_method = shared.wasm_method; let execution = shared.execution; + let heap_pages = if shared.heap_pages.is_some() { + shared.heap_pages + } else { + config.default_heap_pages + }; + let mut changes = Default::default(); - // don't really care about these so we just use the default - let heap_pages = config.default_heap_pages; let max_runtime_instances = config.max_runtime_instances; let executor = NativeExecutor::::new( wasm_method.into(), @@ -242,11 +246,12 @@ where { let wasm_method = shared.wasm_method; let execution = shared.execution; - let heap_pages = if command.heap_pages.is_some() { - command.heap_pages + let heap_pages = if shared.heap_pages.is_some() { + shared.heap_pages } else { config.default_heap_pages }; + let mut changes = Default::default(); let max_runtime_instances = config.max_runtime_instances; let executor = NativeExecutor::::new( From 0d54943da9bb5f0b33976706cba5c737d140188e Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 10:10:13 -0700 Subject: [PATCH 22/52] Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- utils/frame/remote-externalities/src/lib.rs | 1 - utils/frame/try-runtime/cli/src/lib.rs | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index cedda630eaf97..e80dafcdeb08e 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -30,7 +30,6 @@ pub use sp_io::TestExternalities; use sp_core::{ hexdisplay::HexDisplay, storage::{StorageKey, StorageData}, - // offchain::{OffchainWorkerExt, OffchainDbExt, testing::TestOffchainExt}, }; use codec::{Encode, Decode}; use sp_runtime::traits::Block as BlockT; diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 852c0b7d57516..34e7ebe31e72e 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -304,7 +304,7 @@ where builder.build().await? }; - // register externality extensions in order to provide host interface for OCW to the runtime + // register externality extensions in order to provide host interface for OCW to the runtime. let (offchain, _offchain_state) = TestOffchainExt::new(); let (pool, _pool_state) = TestTransactionPoolExt::new(); ext.register_extension(OffchainDbExt::new(offchain.clone())); @@ -313,7 +313,7 @@ where ext.register_extension(TransactionPoolExt::new(pool)); let header_hash: B::Hash = command.header_at.parse().unwrap(); - let header = rpc_api::get_header::(url,header_hash).await; + let header = rpc_api::get_header::(url, header_hash).await; let _ = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, @@ -398,4 +398,3 @@ mod parse { } } } - From d63e364df99bcac86e5f5521dd29cab8238b82bf Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 10:11:23 -0700 Subject: [PATCH 23/52] Remove heap_pages --- primitives/state-machine/src/testing.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 250c2fd4e9a98..df6ca6fb17c9a 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -103,7 +103,6 @@ where assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); assert!(storage.children_default.keys().all(|key| is_child_storage_key(key))); - storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); storage.top.insert(CODE.to_vec(), code.to_vec()); let mut extensions = Extensions::default(); From a27219f3a70d5d6f304061827f73d83bffe4c0e2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 11:28:52 -0700 Subject: [PATCH 24/52] Dry code extraction from state --- Cargo.lock | 1 + primitives/state-machine/src/testing.rs | 40 +++++++++++----------- utils/frame/try-runtime/cli/Cargo.toml | 3 +- utils/frame/try-runtime/cli/src/lib.rs | 45 ++++++++++++------------- 4 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0a6f19968042..c1efb6d57e0de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10661,6 +10661,7 @@ dependencies = [ "log", "parity-scale-codec", "remote-externalities", + "sc-chain-spec", "sc-cli", "sc-client-api", "sc-executor", diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index df6ca6fb17c9a..874e9317d9db5 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -33,32 +33,32 @@ use crate::{ use codec::{Decode, Encode}; use hash_db::Hasher; use sp_core::{ - offchain::testing::TestPersistentOffchainDB, - storage::{ - well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, - Storage, - }, - traits::TaskExecutorExt, - testing::TaskExecutor, + offchain::testing::TestPersistentOffchainDB, + storage::{ + well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, + Storage, + }, + testing::TaskExecutor, + traits::TaskExecutorExt, }; -use sp_externalities::{Extensions, Extension, ExtensionStore}; +use sp_externalities::{Extension, ExtensionStore, Extensions}; /// Simple HashMap-based Externalities impl. pub struct TestExternalities where - H::Out: codec::Codec + Ord, + H::Out: codec::Codec + Ord, { - /// The overlay changed storage. - overlay: OverlayedChanges, - offchain_db: TestPersistentOffchainDB, - storage_transaction_cache: - StorageTransactionCache< as Backend>::Transaction, H, N>, - /// Storage backend. - pub backend: InMemoryBackend, - changes_trie_config: Option, - changes_trie_storage: ChangesTrieInMemoryStorage, - /// Extensions. - pub extensions: Extensions, + /// The overlay changed storage. + overlay: OverlayedChanges, + offchain_db: TestPersistentOffchainDB, + storage_transaction_cache: + StorageTransactionCache< as Backend>::Transaction, H, N>, + /// Storage backend. + pub backend: InMemoryBackend, + changes_trie_config: Option, + changes_trie_storage: ChangesTrieInMemoryStorage, + /// Extensions. + pub extensions: Extensions, } impl TestExternalities diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 86a87a0a9e93a..f262ba4812a0e 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -16,12 +16,13 @@ targets = ["x86_64-unknown-linux-gnu"] log = "0.4.8" parity-scale-codec = { version = "2.0.0" } serde = "1.0.0" +structopt = "0.3.8" sc-service = { version = "0.9.0", default-features = false, path = "../../../../client/service" } sc-cli = { version = "0.9.0", path = "../../../../client/cli" } sc-executor = { version = "0.9.0", path = "../../../../client/executor" } sc-client-api = { version = "3.0.0", path = "../../../../client/api" } -structopt = "0.3.8" +sc-chain-spec = { version = "3.0.0", path = "../../../../client/chain-spec" } sp-state-machine = { version = "0.9.0", path = "../../../../primitives/state-machine" } sp-api = { version = "3.0.0", path = "../../../../primitives/api" } sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 97908225e67c9..ba68db990ff23 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -23,6 +23,7 @@ use sc_service::Configuration; use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; use sc_executor::NativeExecutor; use sc_service::NativeExecutionDispatch; +use sc_chain_spec::ChainSpec; use sp_state_machine::StateMachine; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_core::{ @@ -75,7 +76,7 @@ pub struct SharedParams { value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = "Native", + default_value = "Wasm", )] pub execution: ExecutionStrategy, @@ -149,18 +150,6 @@ where as FromStr>::Err: Debug, ExecDispatch: NativeExecutionDispatch + 'static, { - let spec = config.chain_spec; - let genesis_storage = spec.build_storage()?; - - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); - let code_key = StorageKey(well_known_keys::CODE.to_vec()); - let wasm_method = shared.wasm_method; let execution = shared.execution; let heap_pages = if shared.heap_pages.is_some() { @@ -198,8 +187,9 @@ where ..Default::default() })), }; - + // inject the code into this ext. + let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? }; @@ -294,16 +284,7 @@ where let builder = Builder::::new().mode(mode); let mut ext = if command.overwrite_code { // get the code to inject inject into the ext - let spec = config.chain_spec; // TODO DRY this code block - let genesis_storage = spec.build_storage()?; - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); - let code_key = StorageKey(well_known_keys::CODE.to_vec()); + let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? } else { builder.build().await? @@ -375,6 +356,22 @@ impl CliConfiguration for TryRuntimeCmd { } } +/// Extract `:code` from given chain spec and return as `StorageData` along with the corresponding +/// `StorageKey`. +fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { + let genesis_storage = spec.build_storage()?; + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + + Ok((code_key, code)) +} + // Utils for parsing user input mod parse { pub fn hash(block_number: &str) -> Result { From 8adf25fc7264b1d99193f95a8921e211a32e0078 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 11:33:03 -0700 Subject: [PATCH 25/52] Formatting --- utils/frame/try-runtime/cli/src/lib.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index ba68db990ff23..d025f9ed4f3c0 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -187,7 +187,7 @@ where ..Default::default() })), }; - + // inject the code into this ext. let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? @@ -360,13 +360,7 @@ impl CliConfiguration for TryRuntimeCmd { /// `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { let genesis_storage = spec.build_storage()?; - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); +genesis_storage let code_key = StorageKey(well_known_keys::CODE.to_vec()); Ok((code_key, code)) From 7ed2c481bfe7b23c867061e1d414bd73ad80c6e2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 12:03:27 -0700 Subject: [PATCH 26/52] More formatting --- primitives/state-machine/src/testing.rs | 42 ++++++++++++------------- utils/frame/try-runtime/cli/src/lib.rs | 12 +++++-- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 874e9317d9db5..ea43d391734c6 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -30,35 +30,35 @@ use crate::{ }, }; -use codec::{Decode, Encode}; +use codec::Decode; use hash_db::Hasher; use sp_core::{ - offchain::testing::TestPersistentOffchainDB, - storage::{ - well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, - Storage, - }, - testing::TaskExecutor, - traits::TaskExecutorExt, + offchain::testing::TestPersistentOffchainDB, + storage::{ + well_known_keys::{CHANGES_TRIE_CONFIG, CODE, is_child_storage_key}, + Storage, + }, + traits::TaskExecutorExt, + testing::TaskExecutor, }; -use sp_externalities::{Extension, ExtensionStore, Extensions}; +use sp_externalities::{Extensions, Extension, ExtensionStore}; /// Simple HashMap-based Externalities impl. pub struct TestExternalities where - H::Out: codec::Codec + Ord, + H::Out: codec::Codec + Ord, { - /// The overlay changed storage. - overlay: OverlayedChanges, - offchain_db: TestPersistentOffchainDB, - storage_transaction_cache: - StorageTransactionCache< as Backend>::Transaction, H, N>, - /// Storage backend. - pub backend: InMemoryBackend, - changes_trie_config: Option, - changes_trie_storage: ChangesTrieInMemoryStorage, - /// Extensions. - pub extensions: Extensions, + /// The overlay changed storage. + overlay: OverlayedChanges, + offchain_db: TestPersistentOffchainDB, + storage_transaction_cache: + StorageTransactionCache< as Backend>::Transaction, H, N>, + /// Storage backend. + pub backend: InMemoryBackend, + changes_trie_config: Option, + changes_trie_storage: ChangesTrieInMemoryStorage, + /// Extensions. + pub extensions: Extensions, } impl TestExternalities diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index d025f9ed4f3c0..fccf467044cb7 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -86,7 +86,7 @@ pub struct SharedParams { value_name = "METHOD", possible_values = &WasmExecutionMethod::variants(), case_insensitive = true, - default_value = "compiled" + default_value = "Compiled" )] pub wasm_method: WasmExecutionMethod, @@ -269,7 +269,7 @@ where (Mode::Online(online_config), url) }, State::Snap { snapshot_path } => { - // This is a temporary hack; the url is used just to get the header. We should try + // TODO This is a temporary hack; the url is used just to get the header. We should try // and get the header out of state, OR use an arbitrary header if thats ok, OR allow // the user to feed in a header via file. // This assumes you have a node running on local host default @@ -360,7 +360,13 @@ impl CliConfiguration for TryRuntimeCmd { /// `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { let genesis_storage = spec.build_storage()?; -genesis_storage + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); let code_key = StorageKey(well_known_keys::CODE.to_vec()); Ok((code_key, code)) From 773b1d2dff8f0ea0d44b2dda3d93632079caedcc Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 12:12:32 -0700 Subject: [PATCH 27/52] Add issue todo --- utils/frame/try-runtime/cli/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index fccf467044cb7..e79005f403f38 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -111,7 +111,8 @@ pub struct TryRuntimeCmd { pub enum State { /// Use a state snapshot as the source of runtime state. NOTE: for the offchain-worker command this /// is only partially supported at the moment and you must have a relevant archive node exposed on - //// localhost:9944 in order to query the block header. + /// localhost:9944 in order to query the block header. + // TODO https://github.com/paritytech/substrate/issues/9027 Snap { snapshot_path: PathBuf, }, @@ -272,6 +273,7 @@ where // TODO This is a temporary hack; the url is used just to get the header. We should try // and get the header out of state, OR use an arbitrary header if thats ok, OR allow // the user to feed in a header via file. + // https://github.com/paritytech/substrate/issues/9027 // This assumes you have a node running on local host default let url = "ws://127.0.0.1:9944".to_string(); let mode = Mode::Offline(OfflineConfig { From eec51c5e4afff91f4c1de9878b2d1ea7d92878d3 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 14:15:39 -0700 Subject: [PATCH 28/52] Use jsonrpsee 0.2.0 --- Cargo.lock | 84 ++++++++------------- utils/frame/remote-externalities/Cargo.toml | 6 +- 2 files changed, 34 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1efb6d57e0de..a15ee5218c7eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,19 +351,6 @@ version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" -[[package]] -name = "async-tls" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f23d769dbf1838d5df5156e7b1ad404f4c463d1ac2c6aeb6cd943630f8a8400" -dependencies = [ - "futures-core", - "futures-io", - "rustls 0.19.1", - "webpki 0.21.4", - "webpki-roots", -] - [[package]] name = "async-trait" version = "0.1.48" @@ -2150,7 +2137,7 @@ checksum = "3a1387e07917c711fb4ee4f48ea0adb04a3c9739e53ef85bf43ae1edc2937a8b" dependencies = [ "futures-io", "rustls 0.19.1", - "webpki 0.21.4", + "webpki", ] [[package]] @@ -2665,8 +2652,8 @@ dependencies = [ "rustls 0.18.1", "rustls-native-certs 0.4.0", "tokio 0.2.25", - "tokio-rustls", - "webpki 0.21.4", + "tokio-rustls 0.14.1", + "webpki", ] [[package]] @@ -2999,9 +2986,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.2.0-alpha.7" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e6075694416dfb53fd29910725a4de3dfee2370d4003827d9aafd10e060ef03" +checksum = "3b4c85cfa6767333f3e5f3b2f2f765dad2727b0033ee270ae07c599bf43ed5ae" dependencies = [ "Inflector", "proc-macro-crate 1.0.0", @@ -3012,9 +2999,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.2.0-alpha.7" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60988e2759b2f1cf5b8366db430de14ec54a035659b6bcddefca50d306fe03f" +checksum = "c0cf7bd4e93b3b56e59131de7f24afbea871faf914e97bcdd942c86927ab0172" dependencies = [ "async-trait", "beef", @@ -3030,12 +3017,10 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.2.0-alpha.7" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ac07a90b9fd6de3aedaf839a0d5b38bd481552b0927e567b234e1ed3c1349a4" +checksum = "6ec51150965544e1a4468f372bdab8545243a1b045d4ab272023aac74c60de32" dependencies = [ - "async-std", - "async-tls", "async-trait", "fnv", "futures 0.3.15", @@ -3048,8 +3033,10 @@ dependencies = [ "serde_json", "soketto 0.5.0", "thiserror", + "tokio 0.2.25", + "tokio-rustls 0.15.0", + "tokio-util", "url 2.2.1", - "webpki 0.22.0", ] [[package]] @@ -6806,7 +6793,7 @@ dependencies = [ "sp-core", "sp-io", "sp-runtime", - "tokio 1.6.0", + "tokio 0.2.25", ] [[package]] @@ -6918,7 +6905,7 @@ dependencies = [ "log", "ring", "sct", - "webpki 0.21.4", + "webpki", ] [[package]] @@ -6931,7 +6918,7 @@ dependencies = [ "log", "ring", "sct", - "webpki 0.21.4", + "webpki", ] [[package]] @@ -10195,7 +10182,7 @@ dependencies = [ "pin-project-lite 0.1.12", "signal-hook-registry", "slab", - "tokio-macros 0.2.6", + "tokio-macros", "winapi 0.3.9", ] @@ -10207,7 +10194,6 @@ checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" dependencies = [ "autocfg", "pin-project-lite 0.2.6", - "tokio-macros 1.2.0", ] [[package]] @@ -10285,17 +10271,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-macros" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-named-pipes" version = "0.1.0" @@ -10337,7 +10312,19 @@ dependencies = [ "futures-core", "rustls 0.18.1", "tokio 0.2.25", - "webpki 0.21.4", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d15e5669243a45f630a5167d101b942174ca94b615445b2057eace1c818736" +dependencies = [ + "futures-core", + "rustls 0.19.1", + "tokio 0.2.25", + "webpki", ] [[package]] @@ -10443,6 +10430,7 @@ checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes 0.5.6", "futures-core", + "futures-io", "futures-sink", "log", "pin-project-lite 0.1.12", @@ -11332,23 +11320,13 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" dependencies = [ - "webpki 0.21.4", + "webpki", ] [[package]] diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 7ed7ce3a3c6fc..5709b82efa3ed 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -13,8 +13,8 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee-ws-client = { version = "=0.2.0-alpha.7", default-features = false } -jsonrpsee-proc-macros = "=0.2.0-alpha.7" +jsonrpsee-ws-client = { version = "0.2.0", default-features = false, features = ["tokio02"] } +jsonrpsee-proc-macros = "0.2.0" hex = "0.4.0" env_logger = "0.8.2" @@ -28,7 +28,7 @@ sp-core = { version = "3.0.0", path = "../../../primitives/core" } sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } [dev-dependencies] -tokio = { version = "1.6.0", features = ["macros", "rt"] } +tokio = { version = "0.2", features = ["full"] } pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", version = "4.0.0" } frame-support = { path = "../../../frame/support", version = "3.0.0" } From d394b1e4a98624701f98bb2c106554eeb851f264 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 17:30:26 -0700 Subject: [PATCH 29/52] Try trigger gitlab --- utils/frame/try-runtime/cli/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index e79005f403f38..a8ccef946a277 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -189,7 +189,6 @@ where })), }; - // inject the code into this ext. let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? }; @@ -285,7 +284,6 @@ where }; let builder = Builder::::new().mode(mode); let mut ext = if command.overwrite_code { - // get the code to inject inject into the ext let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? } else { From 0d85d13a9677b8d497da4a437ecbff54fc76906c Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 18:38:36 -0700 Subject: [PATCH 30/52] Fix "block_import_works" test --- frame/executive/src/lib.rs | 2 ++ frame/support/test/tests/derive_no_bound_ui/eq.stderr | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 593b8db92c60d..f064bac43a1df 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -821,7 +821,9 @@ mod tests { } fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { + use sp_core::storage::well_known_keys::HEAP_PAGES; let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + t.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)], }.assimilate_storage(&mut t).unwrap(); diff --git a/frame/support/test/tests/derive_no_bound_ui/eq.stderr b/frame/support/test/tests/derive_no_bound_ui/eq.stderr index 36384178d469b..fce13d6f17f06 100644 --- a/frame/support/test/tests/derive_no_bound_ui/eq.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/eq.stderr @@ -7,6 +7,6 @@ error[E0277]: can't compare `Foo` with `Foo` ::: $RUST/core/src/cmp.rs | | pub trait Eq: PartialEq { - | --------------- required by this bound in `Eq` + | --------------- required by this bound in `std::cmp::Eq` | = help: the trait `PartialEq` is not implemented for `Foo` From 83586b26ee8e27c060e34995f699faacb3fb49a9 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 20:01:19 -0700 Subject: [PATCH 31/52] fix native_big_block_import_fails_on_fallback test --- bin/node/executor/tests/basic.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index fe3ae5f14cc37..236e7b1d1ae6d 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -697,6 +697,8 @@ fn native_big_block_import_succeeds() { fn native_big_block_import_fails_on_fallback() { let mut t = new_test_ext(compact_code_unwrap(), false); + set_heap_pages(&mut t.ext(), 8); + assert!( executor_call:: _>( &mut t, From 59eae13b44b241a64181d2af71a8298ace1d69bf Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 21:06:42 -0700 Subject: [PATCH 32/52] fix commit should work --- primitives/state-machine/src/testing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index ea43d391734c6..363d543da086f 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -307,7 +307,7 @@ mod tests { ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); - let root = H256::from(hex!("2a340d3dfd52f5992c6b117e9e45f479e6da5afffafeb26ab619cf137a95aeb8")); + let root = H256::from(hex!("ed4d8c799d996add422395a6abd7545491d40bd838d738afafa1b8a4de625489")); assert_eq!(H256::from_slice(ext.storage_root().as_slice()), root); } From ada7f670f701c21fb399946a3f6918453f537bcb Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 21:42:16 -0700 Subject: [PATCH 33/52] Rewrite UI tests --- .../tests/pallet_ui/call_argument_invalid_bound_2.stderr | 4 ++-- .../tests/pallet_ui/storage_info_unsatisfied_nmap.stderr | 4 ++-- .../test/tests/ui/impl_incorrect_method_signature.stderr | 4 ++-- .../api/test/tests/ui/mock_only_self_reference.stderr | 8 ++++---- .../ui/type_reference_in_impl_runtime_apis_call.stderr | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr index 1d0e96be9edb9..70ae8e65943c9 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr @@ -4,7 +4,7 @@ error[E0277]: the trait bound `::Bar: WrapperTypeDecode` is 20 | fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^ the trait `WrapperTypeDecode` is not implemented for `::Bar` | - ::: /usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:277:18 + ::: /Users/zeke/.cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:277:18 | 277 | fn decode(input: &mut I) -> Result; | ----- required by this bound in `pallet::_::_parity_scale_codec::Decode::decode` @@ -17,7 +17,7 @@ error[E0277]: the trait bound `::Bar: WrapperTypeEncode` is 20 | fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^ the trait `WrapperTypeEncode` is not implemented for `::Bar` | - ::: /usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:216:21 + ::: /Users/zeke/.cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:216:21 | 216 | fn encode_to(&self, dest: &mut T) { | ------ required by this bound in `encode_to` diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr index 545520124bfee..6c92423c6a7fe 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr @@ -4,6 +4,6 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied 10 | #[pallet::generate_storage_info] | ^^^^^^^^^^^^^^^^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | - = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `NMapKey` - = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, NMapKey, u32>` + = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `Key` + = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, Key, u32>` = note: required by `storage_info` diff --git a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr index fcda69533e3ad..6b00b7268672f 100644 --- a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr +++ b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found struct `std::string::String` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr index 73cf936103798..83cfcf6ca1f9e 100644 --- a/primitives/api/test/tests/ui/mock_only_self_reference.stderr +++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr @@ -24,8 +24,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for trait @@ -42,6 +42,6 @@ error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for t 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr index 71f12b415a2b5..689723f8d7509 100644 --- a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr +++ b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `&u64` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> std::result::Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types From 31b9a31113b3c93528e73691c5e41ead77b07d20 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 22:05:28 -0700 Subject: [PATCH 34/52] Revert "Rewrite UI tests" This reverts commit ada7f670f701c21fb399946a3f6918453f537bcb. --- .../tests/pallet_ui/call_argument_invalid_bound_2.stderr | 4 ++-- .../tests/pallet_ui/storage_info_unsatisfied_nmap.stderr | 4 ++-- .../test/tests/ui/impl_incorrect_method_signature.stderr | 4 ++-- .../api/test/tests/ui/mock_only_self_reference.stderr | 8 ++++---- .../ui/type_reference_in_impl_runtime_apis_call.stderr | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr index 70ae8e65943c9..1d0e96be9edb9 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_2.stderr @@ -4,7 +4,7 @@ error[E0277]: the trait bound `::Bar: WrapperTypeDecode` is 20 | fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^ the trait `WrapperTypeDecode` is not implemented for `::Bar` | - ::: /Users/zeke/.cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:277:18 + ::: /usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:277:18 | 277 | fn decode(input: &mut I) -> Result; | ----- required by this bound in `pallet::_::_parity_scale_codec::Decode::decode` @@ -17,7 +17,7 @@ error[E0277]: the trait bound `::Bar: WrapperTypeEncode` is 20 | fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^ the trait `WrapperTypeEncode` is not implemented for `::Bar` | - ::: /Users/zeke/.cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:216:21 + ::: /usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/parity-scale-codec-2.1.1/src/codec.rs:216:21 | 216 | fn encode_to(&self, dest: &mut T) { | ------ required by this bound in `encode_to` diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr index 6c92423c6a7fe..545520124bfee 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr @@ -4,6 +4,6 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied 10 | #[pallet::generate_storage_info] | ^^^^^^^^^^^^^^^^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | - = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `Key` - = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, Key, u32>` + = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `NMapKey` + = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, NMapKey, u32>` = note: required by `storage_info` diff --git a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr index 6b00b7268672f..fcda69533e3ad 100644 --- a/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr +++ b/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found struct `std::string::String` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr index 83cfcf6ca1f9e..73cf936103798 100644 --- a/primitives/api/test/tests/ui/mock_only_self_reference.stderr +++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr @@ -24,8 +24,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for trait @@ -42,6 +42,6 @@ error[E0053]: method `Api_test2_runtime_api_impl` has an incompatible type for t 12 | sp_api::mock_impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&MockApi, &BlockId, Extrinsic>>, ExecutionContext, Option<()>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr index 689723f8d7509..71f12b415a2b5 100644 --- a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr +++ b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr @@ -23,8 +23,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr 17 | sp_api::impl_runtime_apis! { | ^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `&u64` | - = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> Result<_, _>` - found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> Result<_, _>` + = note: expected fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option, Vec<_>) -> std::result::Result<_, _>` + found fn pointer `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &BlockId<__SR_API_BLOCK__>, ExecutionContext, std::option::Option<&u64>, Vec<_>) -> std::result::Result<_, _>` = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0308]: mismatched types From fad9c5d2ef9dda1a511177f500ad177cf0340d02 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Fri, 4 Jun 2021 22:07:56 -0700 Subject: [PATCH 35/52] try again with UI --- frame/support/test/tests/derive_no_bound_ui/eq.stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/test/tests/derive_no_bound_ui/eq.stderr b/frame/support/test/tests/derive_no_bound_ui/eq.stderr index fce13d6f17f06..36384178d469b 100644 --- a/frame/support/test/tests/derive_no_bound_ui/eq.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/eq.stderr @@ -7,6 +7,6 @@ error[E0277]: can't compare `Foo` with `Foo` ::: $RUST/core/src/cmp.rs | | pub trait Eq: PartialEq { - | --------------- required by this bound in `std::cmp::Eq` + | --------------- required by this bound in `Eq` | = help: the trait `PartialEq` is not implemented for `Foo` From 0e8975b5e011d9456b542b4455f46bdc831eebfc Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 09:52:26 -0700 Subject: [PATCH 36/52] Use const for legacy heap pages val --- bin/node/cli/src/cli.rs | 3 +-- bin/node/executor/tests/basic.rs | 2 +- frame/executive/src/lib.rs | 2 +- primitives/state-machine/src/testing.rs | 4 ++++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index ff1007904d87f..b8ab9267590b3 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,8 +47,7 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// Try some experimental command on the runtime. See `try-runtime --help` for more. - /// NOTE: `try-runtime` feature must be enabled. + /// Try some experimental command on the runtime. Note: `try-runtime` feature must be enabled. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 236e7b1d1ae6d..8b0cedae78698 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -697,7 +697,7 @@ fn native_big_block_import_succeeds() { fn native_big_block_import_fails_on_fallback() { let mut t = new_test_ext(compact_code_unwrap(), false); - set_heap_pages(&mut t.ext(), 8); + set_heap_pages(&mut t.ext(), sp_state_machine::HEAP_PAGES_TEST_LEGACY); assert!( executor_call:: _>( diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index f064bac43a1df..f34d5288bf8ed 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -823,7 +823,7 @@ mod tests { fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { use sp_core::storage::well_known_keys::HEAP_PAGES; let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); + t.top.insert(HEAP_PAGES.to_vec(), sp_state_machine::HEAP_PAGES_TEST_LEGACY); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)], }.assimilate_storage(&mut t).unwrap(); diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 363d543da086f..89fc2914929e0 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -43,6 +43,10 @@ use sp_core::{ }; use sp_externalities::{Extensions, Extension, ExtensionStore}; +/// Legacy default value in `TestExternalities::new_with_code` for `:heap_pages`. +/// `:heap_pages` must now be explicitly set to this value. +pub const HEAP_PAGES_TEST_LEGACY: u64 = 8; + /// Simple HashMap-based Externalities impl. pub struct TestExternalities where From 62a996d93bfedfd198ff03a76b0527db2a02afd1 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:14:33 -0700 Subject: [PATCH 37/52] Move parse module to its own file --- utils/frame/try-runtime/cli/src/lib.rs | 31 ++--------------- utils/frame/try-runtime/cli/src/parse.rs | 44 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 utils/frame/try-runtime/cli/src/parse.rs diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index a8ccef946a277..ba22da47ec2ad 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -36,6 +36,8 @@ use sp_core::{ use sp_keystore::{KeystoreExt, testing::KeyStore}; use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineConfig, rpc_api}; +mod parse; + #[derive(Debug, Clone, structopt::StructOpt)] pub enum Command { OnRuntimeUpgrade(OnRuntimeUpgradeCmd), @@ -371,32 +373,3 @@ fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, Storage Ok((code_key, code)) } - -// Utils for parsing user input -mod parse { - pub fn hash(block_number: &str) -> Result { - let block_number = if block_number.starts_with("0x") { - &block_number[2..] - } else { - block_number - }; - - if let Some(pos) = block_number.chars().position(|c| !c.is_ascii_hexdigit()) { - Err(format!( - "Expected block hash, found illegal hex character at position: {}", - 2 + pos, - )) - } else { - Ok(block_number.into()) - } - } - - pub fn url(s: &str) -> Result { - if s.starts_with("ws://") || s.starts_with("wss://") { - // could use Url crate as well, but lets keep it simple for now. - Ok(s.to_string()) - } else { - Err("not a valid WS(S) url: must start with 'ws://' or 'wss://'") - } - } -} diff --git a/utils/frame/try-runtime/cli/src/parse.rs b/utils/frame/try-runtime/cli/src/parse.rs new file mode 100644 index 0000000000000..296d708c154f5 --- /dev/null +++ b/utils/frame/try-runtime/cli/src/parse.rs @@ -0,0 +1,44 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Utils for parsing user input + +pub(crate) fn hash(block_number: &str) -> Result { + let block_number = if block_number.starts_with("0x") { + &block_number[2..] + } else { + block_number + }; + + if let Some(pos) = block_number.chars().position(|c| !c.is_ascii_hexdigit()) { + Err(format!( + "Expected block hash, found illegal hex character at position: {}", + 2 + pos, + )) + } else { + Ok(block_number.into()) + } +} + +pub(crate) fn url(s: &str) -> Result { + if s.starts_with("ws://") || s.starts_with("wss://") { + // could use Url crate as well, but lets keep it simple for now. + Ok(s.to_string()) + } else { + Err("not a valid WS(S) url: must start with 'ws://' or 'wss://'") + } +} From de5084d16d70fdaa4cd5b4603cc8983f66101517 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:27:23 -0700 Subject: [PATCH 38/52] Move rpc_api module to its own file --- primitives/state-machine/src/testing.rs | 1 + utils/frame/remote-externalities/src/lib.rs | 31 +----------- .../frame/remote-externalities/src/rpc_api.rs | 47 +++++++++++++++++++ 3 files changed, 50 insertions(+), 29 deletions(-) create mode 100644 utils/frame/remote-externalities/src/rpc_api.rs diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 89fc2914929e0..b7cd2d97e7805 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -45,6 +45,7 @@ use sp_externalities::{Extensions, Extension, ExtensionStore}; /// Legacy default value in `TestExternalities::new_with_code` for `:heap_pages`. /// `:heap_pages` must now be explicitly set to this value. +#[allow(dead_code)] pub const HEAP_PAGES_TEST_LEGACY: u64 = 8; /// Simple HashMap-based Externalities impl. diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index e80dafcdeb08e..f3072e52cfb1e 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -37,6 +37,8 @@ use jsonrpsee_ws_client::{ WsClientBuilder, WsClient, v2::params::JsonRpcParams, }; +pub mod rpc_api; + type KeyPair = (StorageKey, StorageData); const LOG_TARGET: &str = "remote-ext"; @@ -449,36 +451,7 @@ impl Builder { } } -/// WS RPC API for one off RPC calls to a substrate node. -/// TODO: Consolidate one off RPC calls https://github.com/paritytech/substrate/issues/8988 -pub mod rpc_api { - use super::*; - /// Get the header of the block identified by `at` - pub async fn get_header>(from: S, at: B::Hash) -> B::Header - where - B::Header: serde::de::DeserializeOwned, - { - use jsonrpsee_ws_client::traits::Client; - let params = vec![serde_json::to_value(at).unwrap()]; - let client = WsClientBuilder::default() - .max_request_body_size(u32::MAX) - .build(from.as_ref()) - .await - .unwrap(); - client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() - } - /// Get the finalized head - pub async fn get_finalized_head>(from: S) -> B::Hash { - use jsonrpsee_ws_client::traits::Client; - let client = WsClientBuilder::default() - .max_request_body_size(u32::MAX) - .build(from.as_ref()) - .await - .unwrap(); - client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() - } -} #[cfg(test)] mod test_prelude { diff --git a/utils/frame/remote-externalities/src/rpc_api.rs b/utils/frame/remote-externalities/src/rpc_api.rs new file mode 100644 index 0000000000000..e67b9f136f708 --- /dev/null +++ b/utils/frame/remote-externalities/src/rpc_api.rs @@ -0,0 +1,47 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! WS RPC API for one off RPC calls to a substrate node. +// TODO: Consolidate one off RPC calls https://github.com/paritytech/substrate/issues/8988 + +use super::*; + +/// Get the header of the block identified by `at` +pub async fn get_header>(from: S, at: B::Hash) -> B::Header +where + B::Header: serde::de::DeserializeOwned, +{ + use jsonrpsee_ws_client::traits::Client; + let params = vec![serde_json::to_value(at).unwrap()]; + let client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(from.as_ref()) + .await + .unwrap(); + client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() +} + +/// Get the finalized head +pub async fn get_finalized_head>(from: S) -> B::Hash { + use jsonrpsee_ws_client::traits::Client; + let client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(from.as_ref()) + .await + .unwrap(); + client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() +} From 2f119f8cc0647f7ced802d69ca93c8a89fc7df99 Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:30:59 -0700 Subject: [PATCH 39/52] Apply suggestions from code review Co-authored-by: Peter Goodspeed-Niklaus --- utils/frame/try-runtime/cli/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index ba22da47ec2ad..8bacfabf9c56c 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -59,7 +59,7 @@ pub struct OffchainWorkerCmd { #[structopt(subcommand)] pub state: State, - /// Wether or not to overwrite the code from state with the code from + /// Whether or not to overwrite the code from state with the code from /// the specified chain spec. #[structopt(long)] pub overwrite_code: bool, From bb15b97e4dde17c65af32f701530b213aeac7a79 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:40:18 -0700 Subject: [PATCH 40/52] trait names: Block, not B --- utils/frame/remote-externalities/src/lib.rs | 2 - utils/frame/try-runtime/cli/src/lib.rs | 62 ++++++++++----------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index f3072e52cfb1e..a77650d042125 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -451,8 +451,6 @@ impl Builder { } } - - #[cfg(test)] mod test_prelude { pub(crate) use super::*; diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 8bacfabf9c56c..d94d3bde111e4 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -140,17 +140,17 @@ pub enum State { }, } -async fn on_runtime_upgrade( +async fn on_runtime_upgrade( shared: SharedParams, command: OnRuntimeUpgradeCmd, config: Configuration ) -> sc_cli::Result<()> where - B: BlockT, - B::Hash: FromStr, - ::Err: Debug, - NumberFor: FromStr, - as FromStr>::Err: Debug, + Block: BlockT, + Block::Hash: FromStr, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, ExecDispatch: NativeExecutionDispatch + 'static, { let wasm_method = shared.wasm_method; @@ -172,7 +172,7 @@ where let ext = { let builder = match command.state { State::Snap { snapshot_path } => { - Builder::::new().mode(Mode::Offline(OfflineConfig { + Builder::::new().mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(snapshot_path), })) }, @@ -181,7 +181,7 @@ where snapshot_path, block_at, modules - } => Builder::::new().mode(Mode::Online(OnlineConfig { + } => Builder::::new().mode(Mode::Online(OnlineConfig { transport: url.to_owned().into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), modules: modules.to_owned().unwrap_or_default(), @@ -195,7 +195,7 @@ where builder.inject(&[(code_key, code)]).build().await? }; - let encoded_result = StateMachine::<_, _, NumberFor, _>::new( + let encoded_result = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, None, &mut changes, @@ -222,18 +222,18 @@ where Ok(()) } -async fn offchain_worker( +async fn offchain_worker( shared: SharedParams, command: OffchainWorkerCmd, config: Configuration, )-> sc_cli::Result<()> where - B: BlockT, - B::Hash: FromStr, - B::Header: serde::de::DeserializeOwned, - ::Err: Debug, - NumberFor: FromStr, - as FromStr>::Err: Debug, + Block: BlockT, + Block::Hash: FromStr, + Block::Header: serde::de::DeserializeOwned, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, ExecDispatch: NativeExecutionDispatch + 'static, { let wasm_method = shared.wasm_method; @@ -284,7 +284,7 @@ where (mode, url) } }; - let builder = Builder::::new().mode(mode); + let builder = Builder::::new().mode(mode); let mut ext = if command.overwrite_code { let (code_key, code) = extract_code(config.chain_spec)?; builder.inject(&[(code_key, code)]).build().await? @@ -300,10 +300,10 @@ where ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()))); ext.register_extension(TransactionPoolExt::new(pool)); - let header_hash: B::Hash = command.header_at.parse().unwrap(); - let header = rpc_api::get_header::(url, header_hash).await; + let header_hash: Block::Hash = command.header_at.parse().unwrap(); + let header = rpc_api::get_header::(url, header_hash).await; - let _ = StateMachine::<_, _, NumberFor, _>::new( + let _ = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, None, &mut changes, @@ -324,22 +324,22 @@ where } impl TryRuntimeCmd { - pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> + pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> where - B: BlockT, - B::Header: serde::de::DeserializeOwned, - B::Hash: FromStr, - ::Err: Debug, - NumberFor: FromStr, - as FromStr>::Err: Debug, + Block: BlockT, + Block::Header: serde::de::DeserializeOwned, + Block::Hash: FromStr, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, ExecDispatch: NativeExecutionDispatch + 'static, { match &self.command { Command::OnRuntimeUpgrade(ref cmd) => { - on_runtime_upgrade::(self.shared.clone(), cmd.clone(), config).await + on_runtime_upgrade::(self.shared.clone(), cmd.clone(), config).await } Command::OffchainWorker(cmd) => { - offchain_worker::(self.shared.clone(), cmd.clone(), config).await + offchain_worker::(self.shared.clone(), cmd.clone(), config).await } } } @@ -358,8 +358,8 @@ impl CliConfiguration for TryRuntimeCmd { } } -/// Extract `:code` from given chain spec and return as `StorageData` along with the corresponding -/// `StorageKey`. +/// Extract `:code` from the given chain spec and return as `StorageData` along with the +/// corresponding `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { let genesis_storage = spec.build_storage()?; let code = StorageData( From 81da802b4d121d03f92d0ad2ff4d665db1f7ae05 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 7 Jun 2021 11:53:34 -0700 Subject: [PATCH 41/52] Corect HEAP_PAGES_TEST_LEGACY export --- Cargo.lock | 1 + frame/executive/Cargo.toml | 1 + frame/executive/src/lib.rs | 4 +++- primitives/state-machine/src/lib.rs | 2 +- primitives/state-machine/src/testing.rs | 1 - 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a15ee5218c7eb..795fc354406bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,6 +1817,7 @@ dependencies = [ "sp-inherents", "sp-io", "sp-runtime", + "sp-state-machine", "sp-std", "sp-tracing", "sp-version", diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index a923f926a0960..2e0fa3f694128 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -31,6 +31,7 @@ pallet-balances = { version = "3.0.0", path = "../balances" } pallet-transaction-payment = { version = "3.0.0", path = "../transaction-payment" } sp-version = { version = "3.0.0", path = "../../primitives/version" } sp-inherents = { version = "3.0.0", path = "../../primitives/inherents" } +sp-state-machine = { version = "0.9.0", default-features = false, path = "../../primitives/state-machine" } [features] default = ["std"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index f34d5288bf8ed..3eea09f3c93f5 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -822,8 +822,10 @@ mod tests { fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { use sp_core::storage::well_known_keys::HEAP_PAGES; + use sp_state_machine::HEAP_PAGES_TEST_LEGACY; + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.top.insert(HEAP_PAGES.to_vec(), sp_state_machine::HEAP_PAGES_TEST_LEGACY); + t.top.insert(HEAP_PAGES.to_vec(), HEAP_PAGES_TEST_LEGACY.encode()); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)], }.assimilate_storage(&mut t).unwrap(); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 479184b4b9905..adce2f91ac166 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -144,7 +144,7 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; - pub use crate::testing::TestExternalities; + pub use crate::testing::{TestExternalities, HEAP_PAGES_TEST_LEGACY}; pub use crate::basic::BasicExternalities; pub use crate::read_only::{ReadOnlyExternalities, InspectState}; pub use crate::changes_trie::{ diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index b7cd2d97e7805..89fc2914929e0 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -45,7 +45,6 @@ use sp_externalities::{Extensions, Extension, ExtensionStore}; /// Legacy default value in `TestExternalities::new_with_code` for `:heap_pages`. /// `:heap_pages` must now be explicitly set to this value. -#[allow(dead_code)] pub const HEAP_PAGES_TEST_LEGACY: u64 = 8; /// Simple HashMap-based Externalities impl. From 8613ed09bc5229089b6a6c716490bf324895c9e5 Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 09:14:09 -0700 Subject: [PATCH 42/52] Update utils/frame/remote-externalities/src/rpc_api.rs Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- utils/frame/remote-externalities/src/rpc_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/rpc_api.rs b/utils/frame/remote-externalities/src/rpc_api.rs index e67b9f136f708..513254ae36150 100644 --- a/utils/frame/remote-externalities/src/rpc_api.rs +++ b/utils/frame/remote-externalities/src/rpc_api.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); From f57f9ae271cfa6bff82789db8748d45bdbd1e035 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 09:51:49 -0700 Subject: [PATCH 43/52] Revert test_ext heap_page insert; adjust storage root instead --- frame/executive/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 3eea09f3c93f5..d8004e14acda1 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -821,11 +821,7 @@ mod tests { } fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - use sp_core::storage::well_known_keys::HEAP_PAGES; - use sp_state_machine::HEAP_PAGES_TEST_LEGACY; - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.top.insert(HEAP_PAGES.to_vec(), HEAP_PAGES_TEST_LEGACY.encode()); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)], }.assimilate_storage(&mut t).unwrap(); @@ -839,7 +835,7 @@ mod tests { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!("6e70de4fa07bac443dc7f8a812c8a0c941aacfa892bb373c5899f7d511d4c25b").into(), + state_root: hex!("ec6bb58b0e4bc7fdf0151a0f601eb825f529fbf90b5be5b2024deba30c5cbbcb").into(), extrinsics_root: hex!("03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314").into(), digest: Digest { logs: vec![], }, }, From d722a45f16b186a9d928ccab131604fbc5797bc7 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 10:40:23 -0700 Subject: [PATCH 44/52] Doc comments for try_runtime::cli::Command --- utils/frame/try-runtime/cli/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index d94d3bde111e4..5c19e211669c2 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -38,9 +38,12 @@ use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineC mod parse; +/// Primary command options for `try-runtime` #[derive(Debug, Clone, structopt::StructOpt)] pub enum Command { + /// Execute "TryRuntime_on_runtime_upgrade" against the given runtime state. OnRuntimeUpgrade(OnRuntimeUpgradeCmd), + /// Execute "OffchainWorkerApi_offchain_worker" against the given runtime state. OffchainWorker(OffchainWorkerCmd), } From d84540a5ddf91193e9c142a055a38bf93349ff91 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 11:46:38 -0700 Subject: [PATCH 45/52] TryRuntime stub --- bin/node/cli/src/cli.rs | 6 +++++- bin/node/cli/src/command.rs | 7 ++++++- utils/frame/try-runtime/cli/src/lib.rs | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index b8ab9267590b3..11ea58f4068df 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,10 +47,14 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// Try some experimental command on the runtime. Note: `try-runtime` feature must be enabled. + /// Try some command against runtime state. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), + /// Try some command against runtime state. Note: `try-runtime` feature must be enabled. + #[cfg(not(feature = "try-runtime"))] + TryRuntime, + /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. Verify(VerifyCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index ece97436bfdf4..18f5433ddf9bb 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -150,7 +150,7 @@ pub fn run() -> Result<()> { }) }, #[cfg(feature = "try-runtime")] - Some(Subcommand::TryRuntime(cmd)) => { + Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { // we don't need any of the components of new_partial, just a runtime, or a task @@ -163,6 +163,11 @@ pub fn run() -> Result<()> { Ok((cmd.run::(config), task_manager)) }) + }, + #[cfg(not(feature = "try-runtime"))] + Some(Subcommand::TryRuntime) => { + Err("TryRuntime wasn't enabled when building the node. \ + You can enable it with `--features try-runtime`.".into()) } } } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 5c19e211669c2..7209ee614007d 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -38,7 +38,7 @@ use remote_externalities::{Builder, Mode, SnapshotConfig, OfflineConfig, OnlineC mod parse; -/// Primary command options for `try-runtime` +/// Possible subcommands of `try-runtime`. #[derive(Debug, Clone, structopt::StructOpt)] pub enum Command { /// Execute "TryRuntime_on_runtime_upgrade" against the given runtime state. From 4808572f8073bdf6e0d9706b1a8b4006ec8379a6 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 12:16:49 -0700 Subject: [PATCH 46/52] trailing comma --- bin/node/cli/src/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 18f5433ddf9bb..3350328609801 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -168,6 +168,6 @@ pub fn run() -> Result<()> { Some(Subcommand::TryRuntime) => { Err("TryRuntime wasn't enabled when building the node. \ You can enable it with `--features try-runtime`.".into()) - } + }, } } From dc3041dd8cdae7331c54e52366c604c753f0b9fe Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 14:33:48 -0700 Subject: [PATCH 47/52] Remove unused dev dep in frame-executive --- Cargo.lock | 1 - frame/executive/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28b0a76dac222..a6c7873f6f066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1817,7 +1817,6 @@ dependencies = [ "sp-inherents", "sp-io", "sp-runtime", - "sp-state-machine", "sp-std", "sp-tracing", "sp-version", diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 2e0fa3f694128..a923f926a0960 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -31,7 +31,6 @@ pallet-balances = { version = "3.0.0", path = "../balances" } pallet-transaction-payment = { version = "3.0.0", path = "../transaction-payment" } sp-version = { version = "3.0.0", path = "../../primitives/version" } sp-inherents = { version = "3.0.0", path = "../../primitives/inherents" } -sp-state-machine = { version = "0.9.0", default-features = false, path = "../../primitives/state-machine" } [features] default = ["std"] From 894df17ef5f3227a6f629f1b70ef52da9047cae3 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 14:51:05 -0700 Subject: [PATCH 48/52] Improve parse::hash variable name & error index --- bin/node/cli/src/command.rs | 2 +- utils/frame/try-runtime/cli/src/parse.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 3350328609801..1ef1da6ba6819 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -150,7 +150,7 @@ pub fn run() -> Result<()> { }) }, #[cfg(feature = "try-runtime")] - Some(Subcommand::TryRuntime(cmd)) => { + Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { // we don't need any of the components of new_partial, just a runtime, or a task diff --git a/utils/frame/try-runtime/cli/src/parse.rs b/utils/frame/try-runtime/cli/src/parse.rs index 296d708c154f5..beb9a6508fed1 100644 --- a/utils/frame/try-runtime/cli/src/parse.rs +++ b/utils/frame/try-runtime/cli/src/parse.rs @@ -17,20 +17,20 @@ //! Utils for parsing user input -pub(crate) fn hash(block_number: &str) -> Result { - let block_number = if block_number.starts_with("0x") { - &block_number[2..] +pub(crate) fn hash(block_hash: &str) -> Result { + let (block_hash, offset) = if block_hash.starts_with("0x") { + (&block_hash[2..], 2) } else { - block_number + (block_hash, 0) }; - if let Some(pos) = block_number.chars().position(|c| !c.is_ascii_hexdigit()) { + if let Some(pos) = block_hash.chars().position(|c| !c.is_ascii_hexdigit()) { Err(format!( "Expected block hash, found illegal hex character at position: {}", - 2 + pos, + offset + pos, )) } else { - Ok(block_number.into()) + Ok(block_hash.into()) } } From 60123c9fd30251fb4aafc9e8fcaa91594578b41e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:11:30 -0700 Subject: [PATCH 49/52] Use Result for rpc_api fns --- .../frame/remote-externalities/src/rpc_api.rs | 21 ++++++++++++------- utils/frame/try-runtime/cli/src/lib.rs | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/utils/frame/remote-externalities/src/rpc_api.rs b/utils/frame/remote-externalities/src/rpc_api.rs index 513254ae36150..3ce7da0cfdd17 100644 --- a/utils/frame/remote-externalities/src/rpc_api.rs +++ b/utils/frame/remote-externalities/src/rpc_api.rs @@ -20,28 +20,35 @@ use super::*; +const WS_CLIENT_BUILDER_ERR: &str = "`jsonrpsee::WsClientBuilder` failed to build."; + /// Get the header of the block identified by `at` -pub async fn get_header>(from: S, at: B::Hash) -> B::Header +pub async fn get_header>(from: S, at: B::Hash) -> Result where B::Header: serde::de::DeserializeOwned, { use jsonrpsee_ws_client::traits::Client; - let params = vec![serde_json::to_value(at).unwrap()]; + let at = serde_json::to_value(at).map_err(|_| "Block hash could not be converted to JSON.")?; + let params = vec![at]; let client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(from.as_ref()) .await - .unwrap(); - client.request::("chain_getHeader", JsonRpcParams::Array(params)).await.unwrap() + .map_err(|_| WS_CLIENT_BUILDER_ERR)?; + client.request::("chain_getHeader", JsonRpcParams::Array(params)) + .await + .map_err(|_| "chain_getHeader request failed.") } /// Get the finalized head -pub async fn get_finalized_head>(from: S) -> B::Hash { +pub async fn get_finalized_head>(from: S) -> Result { use jsonrpsee_ws_client::traits::Client; let client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(from.as_ref()) .await - .unwrap(); - client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams).await.unwrap() + .map_err(|_| WS_CLIENT_BUILDER_ERR)?; + client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams) + .await + .map_err(|_| "chain_getFinalizedHead request failed.") } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 7209ee614007d..c41afdd8106fc 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -304,7 +304,7 @@ where ext.register_extension(TransactionPoolExt::new(pool)); let header_hash: Block::Hash = command.header_at.parse().unwrap(); - let header = rpc_api::get_header::(url, header_hash).await; + let header = rpc_api::get_header::(url, header_hash).await?; let _ = StateMachine::<_, _, NumberFor, _>::new( &ext.backend, From 699ea3f3ecd580800911f7b3b1a5d8912b5f0eae Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:36:18 -0700 Subject: [PATCH 50/52] Richer err messagges --- utils/frame/remote-externalities/src/rpc_api.rs | 17 ++++++++--------- utils/frame/try-runtime/cli/src/lib.rs | 4 +++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/utils/frame/remote-externalities/src/rpc_api.rs b/utils/frame/remote-externalities/src/rpc_api.rs index 3ce7da0cfdd17..e7fd021bac4a8 100644 --- a/utils/frame/remote-externalities/src/rpc_api.rs +++ b/utils/frame/remote-externalities/src/rpc_api.rs @@ -20,35 +20,34 @@ use super::*; -const WS_CLIENT_BUILDER_ERR: &str = "`jsonrpsee::WsClientBuilder` failed to build."; - /// Get the header of the block identified by `at` -pub async fn get_header>(from: S, at: B::Hash) -> Result +pub async fn get_header>(from: S, at: B::Hash) -> Result where B::Header: serde::de::DeserializeOwned, { use jsonrpsee_ws_client::traits::Client; - let at = serde_json::to_value(at).map_err(|_| "Block hash could not be converted to JSON.")?; + let at = serde_json::to_value(at) + .map_err(|e| format!("Block hash could not be converted to JSON due to {:?}", e))?; let params = vec![at]; let client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(from.as_ref()) .await - .map_err(|_| WS_CLIENT_BUILDER_ERR)?; + .map_err(|e| format!("`WsClientBuilder` failed to build do to {:?}", e))?; client.request::("chain_getHeader", JsonRpcParams::Array(params)) .await - .map_err(|_| "chain_getHeader request failed.") + .map_err(|e| format!("chain_getHeader request failed due to {:?}", e)) } /// Get the finalized head -pub async fn get_finalized_head>(from: S) -> Result { +pub async fn get_finalized_head>(from: S) -> Result { use jsonrpsee_ws_client::traits::Client; let client = WsClientBuilder::default() .max_request_body_size(u32::MAX) .build(from.as_ref()) .await - .map_err(|_| WS_CLIENT_BUILDER_ERR)?; + .map_err(|e| format!("`WsClientBuilder` failed to build do to {:?}", e))?; client.request::("chain_getFinalizedHead", JsonRpcParams::NoParams) .await - .map_err(|_| "chain_getFinalizedHead request failed.") + .map_err(|e| format!("chain_getFinalizedHead request failed due to {:?}", e)) } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index c41afdd8106fc..dc4cb7cd33dbd 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -303,7 +303,9 @@ where ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()))); ext.register_extension(TransactionPoolExt::new(pool)); - let header_hash: Block::Hash = command.header_at.parse().unwrap(); + let header_hash: Block::Hash = command.header_at + .parse() + .map_err(|e| format!("Could not parse header hash: {:?}", e))?; let header = rpc_api::get_header::(url, header_hash).await?; let _ = StateMachine::<_, _, NumberFor, _>::new( From 485cd837685e82dd75fb23cce59de134615b489a Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:20:25 -0700 Subject: [PATCH 51/52] Remove HEAP_PAGE_TEST_LEGACY --- bin/node/executor/tests/basic.rs | 4 +++- primitives/state-machine/src/lib.rs | 2 +- primitives/state-machine/src/testing.rs | 4 ---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 8b0cedae78698..c37260e402565 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -697,7 +697,9 @@ fn native_big_block_import_succeeds() { fn native_big_block_import_fails_on_fallback() { let mut t = new_test_ext(compact_code_unwrap(), false); - set_heap_pages(&mut t.ext(), sp_state_machine::HEAP_PAGES_TEST_LEGACY); + // We set the heap pages to 8 because we know that should give an OOM in WASM with the given block. + // Note: 8 is here for legacy reasons. + set_heap_pages(&mut t.ext(), 8); assert!( executor_call:: _>( diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 920fe9e95b98c..0508bfb780929 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -144,7 +144,7 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; - pub use crate::testing::{TestExternalities, HEAP_PAGES_TEST_LEGACY}; + pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; pub use crate::read_only::{ReadOnlyExternalities, InspectState}; pub use crate::changes_trie::{ diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 89fc2914929e0..363d543da086f 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -43,10 +43,6 @@ use sp_core::{ }; use sp_externalities::{Extensions, Extension, ExtensionStore}; -/// Legacy default value in `TestExternalities::new_with_code` for `:heap_pages`. -/// `:heap_pages` must now be explicitly set to this value. -pub const HEAP_PAGES_TEST_LEGACY: u64 = 8; - /// Simple HashMap-based Externalities impl. pub struct TestExternalities where From b792e5ccaa18f44511cc8d2f77f0bd1e7b8f45b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 9 Jun 2021 22:01:51 +0200 Subject: [PATCH 52/52] Update bin/node/executor/tests/basic.rs --- bin/node/executor/tests/basic.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index c37260e402565..56e6aa4dde993 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -698,7 +698,6 @@ fn native_big_block_import_fails_on_fallback() { let mut t = new_test_ext(compact_code_unwrap(), false); // We set the heap pages to 8 because we know that should give an OOM in WASM with the given block. - // Note: 8 is here for legacy reasons. set_heap_pages(&mut t.ext(), 8); assert!(