diff --git a/cli/src/commands/codegen.rs b/cli/src/commands/codegen.rs index 6a3366defc..d7f7a38d21 100644 --- a/cli/src/commands/codegen.rs +++ b/cli/src/commands/codegen.rs @@ -4,12 +4,7 @@ use clap::Parser as ClapParser; use color_eyre::eyre; -use frame_metadata::RuntimeMetadataPrefixed; use jsonrpsee::client_transport::ws::Uri; -use scale::{ - Decode, - Input, -}; use std::{ fs, io::Read, @@ -48,7 +43,7 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> { let mut file = fs::File::open(file)?; let mut bytes = Vec::new(); file.read_to_end(&mut bytes)?; - codegen(&mut &bytes[..], opts.derives, opts.crate_path)?; + codegen(&bytes, opts.derives, opts.crate_path)?; return Ok(()) } @@ -57,18 +52,16 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> { .parse::() .expect("default url is valid") }); - let (_, bytes) = super::metadata::fetch_metadata(&url).await?; - codegen(&mut &bytes[..], opts.derives, opts.crate_path)?; + let bytes = subxt_codegen::utils::fetch_metadata_bytes(&url).await?; + codegen(&bytes, opts.derives, opts.crate_path)?; Ok(()) } -fn codegen( - encoded: &mut I, +fn codegen( + metadata_bytes: &[u8], raw_derives: Vec, crate_path: Option, ) -> color_eyre::Result<()> { - let metadata = ::decode(encoded)?; - let generator = subxt_codegen::RuntimeGenerator::new(metadata); let item_mod = syn::parse_quote!( pub mod api {} ); @@ -82,7 +75,12 @@ fn codegen( let mut derives = DerivesRegistry::new(&crate_path); derives.extend_for_all(p.into_iter()); - let runtime_api = generator.generate_runtime(item_mod, derives, crate_path); + let runtime_api = subxt_codegen::generate_runtime_api_from_bytes( + item_mod, + metadata_bytes, + derives, + crate_path, + ); println!("{}", runtime_api); Ok(()) } diff --git a/cli/src/commands/compatibility.rs b/cli/src/commands/compatibility.rs index fcf73c5ef2..3071c5b2f5 100644 --- a/cli/src/commands/compatibility.rs +++ b/cli/src/commands/compatibility.rs @@ -111,7 +111,7 @@ async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> { } async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result { - let (_, bytes) = super::metadata::fetch_metadata(url).await?; + let bytes = subxt_codegen::utils::fetch_metadata_bytes(url).await?; let metadata = ::decode(&mut &bytes[..])?; if metadata.0 != META_RESERVED { diff --git a/cli/src/commands/metadata.rs b/cli/src/commands/metadata.rs index a2f9890bc3..b403e2edb9 100644 --- a/cli/src/commands/metadata.rs +++ b/cli/src/commands/metadata.rs @@ -5,24 +5,13 @@ use clap::Parser as ClapParser; use color_eyre::eyre; use frame_metadata::RuntimeMetadataPrefixed; -use jsonrpsee::{ - async_client::ClientBuilder, - client_transport::ws::{ - Uri, - WsTransportClientBuilder, - }, - core::{ - client::ClientT, - Error, - }, - http_client::HttpClientBuilder, - rpc_params, -}; +use jsonrpsee::client_transport::ws::Uri; use scale::Decode; use std::io::{ self, Write, }; +use subxt_codegen::utils::fetch_metadata_hex; /// Download metadata from a substrate node, for use with `subxt` codegen. #[derive(Debug, ClapParser)] @@ -41,10 +30,11 @@ pub struct Opts { } pub async fn run(opts: Opts) -> color_eyre::Result<()> { - let (hex_data, bytes) = fetch_metadata(&opts.url).await?; + let hex_data = fetch_metadata_hex(&opts.url).await?; match opts.format.as_str() { "json" => { + let bytes = hex::decode(hex_data.trim_start_matches("0x"))?; let metadata = ::decode(&mut &bytes[..])?; let json = serde_json::to_string_pretty(&metadata)?; println!("{}", json); @@ -54,7 +44,10 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> { println!("{}", hex_data); Ok(()) } - "bytes" => Ok(io::stdout().write_all(&bytes)?), + "bytes" => { + let bytes = hex::decode(hex_data.trim_start_matches("0x"))?; + Ok(io::stdout().write_all(&bytes)?) + } _ => { Err(eyre::eyre!( "Unsupported format `{}`, expected `json`, `hex` or `bytes`", @@ -63,40 +56,3 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> { } } } - -pub async fn fetch_metadata(url: &Uri) -> color_eyre::Result<(String, Vec)> { - let hex_data = match url.scheme_str() { - Some("http") => fetch_metadata_http(url).await, - Some("ws") | Some("wss") => fetch_metadata_ws(url).await, - invalid_scheme => { - let scheme = invalid_scheme.unwrap_or("no scheme"); - Err(eyre::eyre!(format!( - "`{}` not supported, expects 'http', 'ws', or 'wss'", - scheme - ))) - } - }?; - - let bytes = hex::decode(hex_data.trim_start_matches("0x"))?; - - Ok((hex_data, bytes)) -} - -async fn fetch_metadata_ws(url: &Uri) -> color_eyre::Result { - let (sender, receiver) = WsTransportClientBuilder::default() - .build(url.to_string().parse::().unwrap()) - .await - .map_err(|e| Error::Transport(e.into()))?; - - let client = ClientBuilder::default() - .max_notifs_per_subscription(4096) - .build_with_tokio(sender, receiver); - - Ok(client.request("state_getMetadata", rpc_params![]).await?) -} - -async fn fetch_metadata_http(url: &Uri) -> color_eyre::Result { - let client = HttpClientBuilder::default().build(url.to_string())?; - - Ok(client.request::("state_getMetadata", None).await?) -} diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index fa3d927ac2..1996ce36da 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -21,6 +21,9 @@ quote = "1.0.8" syn = "1.0.58" scale-info = { version = "2.0.0", features = ["bit-vec"] } subxt-metadata = { version = "0.24.0", path = "../metadata" } +jsonrpsee = { version = "0.15.1", features = ["async-client", "client-ws-transport", "http-client"] } +hex = "0.4.3" +tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] } [dev-dependencies] bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } diff --git a/codegen/src/api/mod.rs b/codegen/src/api/mod.rs index 9b1530bb1b..df31e4ad02 100644 --- a/codegen/src/api/mod.rs +++ b/codegen/src/api/mod.rs @@ -19,6 +19,10 @@ use crate::{ CompositeDefFields, TypeGenerator, }, + utils::{ + fetch_metadata_bytes_blocking, + Uri, + }, CratePath, }; use codec::Decode; @@ -50,9 +54,10 @@ use syn::parse_quote; /// * `item_mod` - The module declaration for which the API is implemented. /// * `path` - The path to the scale encoded metadata of the runtime node. /// * `derives` - Provide custom derives for the generated types. +/// * `crate_path` - Path to the `subxt` crate. /// /// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases. -pub fn generate_runtime_api

( +pub fn generate_runtime_api_from_path

( item_mod: syn::ItemMod, path: P, derives: DerivesRegistry, @@ -69,6 +74,49 @@ where file.read_to_end(&mut bytes) .unwrap_or_else(|e| abort_call_site!("Failed to read metadata file: {}", e)); + generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path) +} + +/// Generates the API for interacting with a substrate runtime, using metadata +/// that can be downloaded from a node at the provided URL. This function blocks +/// while retrieving the metadata. +/// +/// # Arguments +/// +/// * `item_mod` - The module declaration for which the API is implemented. +/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from. +/// * `derives` - Provide custom derives for the generated types. +/// * `crate_path` - Path to the `subxt` crate. +/// +/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases. +pub fn generate_runtime_api_from_url( + item_mod: syn::ItemMod, + url: &Uri, + derives: DerivesRegistry, + crate_path: CratePath, +) -> TokenStream2 { + let bytes = fetch_metadata_bytes_blocking(url) + .unwrap_or_else(|e| abort_call_site!("Failed to obtain metadata: {}", e)); + + generate_runtime_api_from_bytes(item_mod, &bytes, derives, crate_path) +} + +/// Generates the API for interacting with a substrate runtime, using metadata bytes. +/// +/// # Arguments +/// +/// * `item_mod` - The module declaration for which the API is implemented. +/// * `url` - HTTP/WS URL to the substrate node you'd like to pull metadata from. +/// * `derives` - Provide custom derives for the generated types. +/// * `crate_path` - Path to the `subxt` crate. +/// +/// **Note:** This is a wrapper over [RuntimeGenerator] for static metadata use-cases. +pub fn generate_runtime_api_from_bytes( + item_mod: syn::ItemMod, + bytes: &[u8], + derives: DerivesRegistry, + crate_path: CratePath, +) -> TokenStream2 { let metadata = frame_metadata::RuntimeMetadataPrefixed::decode(&mut &bytes[..]) .unwrap_or_else(|e| abort_call_site!("Failed to decode metadata: {}", e)); @@ -84,8 +132,9 @@ pub struct RuntimeGenerator { impl RuntimeGenerator { /// Create a new runtime generator from the provided metadata. /// - /// **Note:** If you have a path to the metadata, prefer to use [generate_runtime_api] - /// for generating the runtime API. + /// **Note:** If you have the metadata path, URL or bytes to hand, prefer to use + /// one of the `generate_runtime_api_from_*` functions for generating the runtime API + /// from that. pub fn new(metadata: RuntimeMetadataPrefixed) -> Self { match metadata.1 { RuntimeMetadata::V14(v14) => Self { metadata: v14 }, diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 0a58906f68..bd87e4f21d 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -45,9 +45,13 @@ mod api; mod ir; mod types; +pub mod utils; + pub use self::{ api::{ - generate_runtime_api, + generate_runtime_api_from_bytes, + generate_runtime_api_from_path, + generate_runtime_api_from_url, RuntimeGenerator, }, types::{ diff --git a/codegen/src/utils/fetch_metadata.rs b/codegen/src/utils/fetch_metadata.rs new file mode 100644 index 0000000000..f417aa3e32 --- /dev/null +++ b/codegen/src/utils/fetch_metadata.rs @@ -0,0 +1,108 @@ +use jsonrpsee::{ + async_client::ClientBuilder, + client_transport::ws::{ + Uri, + WsTransportClientBuilder, + }, + core::{ + client::ClientT, + Error, + }, + http_client::HttpClientBuilder, + rpc_params, +}; + +/// Returns the metadata bytes from the provided URL, blocking the current thread. +pub fn fetch_metadata_bytes_blocking(url: &Uri) -> Result, FetchMetadataError> { + tokio_block_on(fetch_metadata_bytes(url)) +} + +/// Returns the raw, 0x prefixed metadata hex from the provided URL, blocking the current thread. +pub fn fetch_metadata_hex_blocking(url: &Uri) -> Result { + tokio_block_on(fetch_metadata_hex(url)) +} + +// Block on some tokio runtime for sync contexts +fn tokio_block_on>(fut: Fut) -> T { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(fut) +} + +/// Returns the metadata bytes from the provided URL. +pub async fn fetch_metadata_bytes(url: &Uri) -> Result, FetchMetadataError> { + let hex = fetch_metadata_hex(url).await?; + let bytes = hex::decode(hex.trim_start_matches("0x"))?; + Ok(bytes) +} + +/// Returns the raw, 0x prefixed metadata hex from the provided URL. +pub async fn fetch_metadata_hex(url: &Uri) -> Result { + let hex_data = match url.scheme_str() { + Some("http") | Some("https") => fetch_metadata_http(url).await, + Some("ws") | Some("wss") => fetch_metadata_ws(url).await, + invalid_scheme => { + let scheme = invalid_scheme.unwrap_or("no scheme"); + Err(FetchMetadataError::InvalidScheme(scheme.to_owned())) + } + }?; + Ok(hex_data) +} + +async fn fetch_metadata_ws(url: &Uri) -> Result { + let (sender, receiver) = WsTransportClientBuilder::default() + .build(url.to_string().parse::().unwrap()) + .await + .map_err(|e| Error::Transport(e.into()))?; + + let client = ClientBuilder::default() + .max_notifs_per_subscription(4096) + .build_with_tokio(sender, receiver); + + Ok(client.request("state_getMetadata", rpc_params![]).await?) +} + +async fn fetch_metadata_http(url: &Uri) -> Result { + let client = HttpClientBuilder::default().build(url.to_string())?; + + Ok(client.request::("state_getMetadata", None).await?) +} + +#[derive(Debug)] +pub enum FetchMetadataError { + DecodeError(hex::FromHexError), + RequestError(jsonrpsee::core::Error), + InvalidScheme(String), +} + +impl std::fmt::Display for FetchMetadataError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FetchMetadataError::DecodeError(e) => { + write!(f, "Cannot decode hex value: {e}") + } + FetchMetadataError::RequestError(e) => write!(f, "Request error: {e}"), + FetchMetadataError::InvalidScheme(s) => { + write!( + f, + "'{s}' not supported, supported URI schemes are http, https, ws or wss." + ) + } + } + } +} + +impl std::error::Error for FetchMetadataError {} + +impl From for FetchMetadataError { + fn from(e: hex::FromHexError) -> Self { + FetchMetadataError::DecodeError(e) + } +} +impl From for FetchMetadataError { + fn from(e: jsonrpsee::core::Error) -> Self { + FetchMetadataError::RequestError(e) + } +} diff --git a/codegen/src/utils/mod.rs b/codegen/src/utils/mod.rs new file mode 100644 index 0000000000..0e29fc7561 --- /dev/null +++ b/codegen/src/utils/mod.rs @@ -0,0 +1,12 @@ +mod fetch_metadata; + +// easy access to this type needed for fetching metadata: +pub use jsonrpsee::client_transport::ws::Uri; + +pub use fetch_metadata::{ + fetch_metadata_bytes, + fetch_metadata_bytes_blocking, + fetch_metadata_hex, + fetch_metadata_hex_blocking, + FetchMetadataError, +}; diff --git a/examples/examples/custom_metadata_url.rs b/examples/examples/custom_metadata_url.rs new file mode 100644 index 0000000000..7123695c45 --- /dev/null +++ b/examples/examples/custom_metadata_url.rs @@ -0,0 +1,12 @@ +// Copyright 2019-2022 Parity Technologies (UK) Ltd. +// This file is dual-licensed as Apache-2.0 or GPL-3.0. +// see LICENSE for license details. + +// If you'd like to use metadata directly from a running node, you +// can provide a URL to that node here. HTTP or WebSocket URLs can be +// provided. Note that if the metadata cannot be retrieved from this +// node URL at compile time, compilation will fail. +#[subxt::subxt(runtime_metadata_url = "wss://rpc.polkadot.io:443")] +pub mod polkadot {} + +fn main() {} diff --git a/macro/src/lib.rs b/macro/src/lib.rs index 28612786e4..cd685d6c38 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -89,10 +89,18 @@ extern crate proc_macro; +use std::str::FromStr; + use darling::FromMeta; use proc_macro::TokenStream; -use proc_macro_error::proc_macro_error; -use subxt_codegen::DerivesRegistry; +use proc_macro_error::{ + abort_call_site, + proc_macro_error, +}; +use subxt_codegen::{ + utils::Uri, + DerivesRegistry, +}; use syn::{ parse_macro_input, punctuated::Punctuated, @@ -100,7 +108,10 @@ use syn::{ #[derive(Debug, FromMeta)] struct RuntimeMetadataArgs { - runtime_metadata_path: String, + #[darling(default)] + runtime_metadata_path: Option, + #[darling(default)] + runtime_metadata_url: Option, #[darling(default)] derive_for_all_types: Option>, #[darling(multiple)] @@ -126,10 +137,6 @@ pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream { Err(e) => return TokenStream::from(e.write_errors()), }; - let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); - let root_path = std::path::Path::new(&root); - let path = root_path.join(args.runtime_metadata_path); - let crate_path = match args.crate_path { Some(crate_path) => crate_path.into(), None => subxt_codegen::CratePath::default(), @@ -146,6 +153,36 @@ pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream { ) } - subxt_codegen::generate_runtime_api(item_mod, &path, derives_registry, crate_path) - .into() + match (args.runtime_metadata_path, args.runtime_metadata_url) { + (Some(rest_of_path), None) => { + let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); + let root_path = std::path::Path::new(&root); + let path = root_path.join(rest_of_path); + subxt_codegen::generate_runtime_api_from_path( + item_mod, + &path, + derives_registry, + crate_path, + ) + .into() + } + (None, Some(url_string)) => { + let url = Uri::from_str(&url_string).unwrap_or_else(|_| { + abort_call_site!("Cannot download metadata; invalid url: {}", url_string) + }); + subxt_codegen::generate_runtime_api_from_url( + item_mod, + &url, + derives_registry, + crate_path, + ) + .into() + } + (None, None) => { + abort_call_site!("One of 'runtime_metadata_path' or 'runtime_metadata_url' must be provided") + } + (Some(_), Some(_)) => { + abort_call_site!("Only one of 'runtime_metadata_path' or 'runtime_metadata_url' can be provided") + } + } } diff --git a/testing/ui-tests/src/incorrect/need_url_or_path.rs b/testing/ui-tests/src/incorrect/need_url_or_path.rs new file mode 100644 index 0000000000..667c75bfc6 --- /dev/null +++ b/testing/ui-tests/src/incorrect/need_url_or_path.rs @@ -0,0 +1,4 @@ +#[subxt::subxt()] +pub mod node_runtime {} + +fn main() {} diff --git a/testing/ui-tests/src/incorrect/need_url_or_path.stderr b/testing/ui-tests/src/incorrect/need_url_or_path.stderr new file mode 100644 index 0000000000..afdcbcf213 --- /dev/null +++ b/testing/ui-tests/src/incorrect/need_url_or_path.stderr @@ -0,0 +1,7 @@ +error: One of 'runtime_metadata_path' or 'runtime_metadata_url' must be provided + --> src/incorrect/need_url_or_path.rs:1:1 + | +1 | #[subxt::subxt()] + | ^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/testing/ui-tests/src/incorrect/url_and_path_provided.rs b/testing/ui-tests/src/incorrect/url_and_path_provided.rs new file mode 100644 index 0000000000..a6561cabf1 --- /dev/null +++ b/testing/ui-tests/src/incorrect/url_and_path_provided.rs @@ -0,0 +1,7 @@ +#[subxt::subxt( + runtime_metadata_path = "../../../artifacts/polkadot_metadata.scale", + runtime_metadata_url = "wss://rpc.polkadot.io:443" +)] +pub mod node_runtime {} + +fn main() {} diff --git a/testing/ui-tests/src/incorrect/url_and_path_provided.stderr b/testing/ui-tests/src/incorrect/url_and_path_provided.stderr new file mode 100644 index 0000000000..a680ed2492 --- /dev/null +++ b/testing/ui-tests/src/incorrect/url_and_path_provided.stderr @@ -0,0 +1,10 @@ +error: Only one of 'runtime_metadata_path' or 'runtime_metadata_url' can be provided + --> src/incorrect/url_and_path_provided.rs:1:1 + | +1 | / #[subxt::subxt( +2 | | runtime_metadata_path = "../../../artifacts/polkadot_metadata.scale", +3 | | runtime_metadata_url = "wss://rpc.polkadot.io:443" +4 | | )] + | |__^ + | + = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info)