Refactor CLI tool to give room for growth (#667)

* refactor CLI commands for easier expansion

* add license headers

* cargo fmt

cli/Cargo.toml

@@ -20,7 +20,7 @@ subxt-codegen = { version = "0.24.0", path = "../codegen" }
 # perform node compatibility
 subxt-metadata = { version = "0.24.0", path = "../metadata" }
 # parse command line args
-structopt = "0.3.25"
+clap = { version = "3.2.22", features = ["derive"] }
 # colourful error reports
 color-eyre = "0.6.1"
 # serialize the metadata

cli/src/commands/codegen.rs

@@ -0,0 +1,88 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use clap::Parser as ClapParser;
+use color_eyre::eyre;
+use frame_metadata::RuntimeMetadataPrefixed;
+use jsonrpsee::client_transport::ws::Uri;
+use scale::{
+    Decode,
+    Input,
+};
+use std::{
+    fs,
+    io::Read,
+    path::PathBuf,
+};
+use subxt_codegen::DerivesRegistry;
+
+/// Generate runtime API client code from metadata.
+///
+/// # Example (with code formatting)
+///
+/// `subxt codegen | rustfmt --edition=2018 --emit=stdout`
+#[derive(Debug, ClapParser)]
+pub struct Opts {
+    /// The url of the substrate node to query for metadata for codegen.
+    #[clap(name = "url", long, parse(try_from_str))]
+    url: Option<Uri>,
+    /// The path to the encoded metadata file.
+    #[clap(short, long, parse(from_os_str))]
+    file: Option<PathBuf>,
+    /// Additional derives
+    #[clap(long = "derive")]
+    derives: Vec<String>,
+    /// The `subxt` crate access path in the generated code.
+    /// Defaults to `::subxt`.
+    #[clap(long = "crate")]
+    crate_path: Option<String>,
+}
+
+pub async fn run(opts: Opts) -> color_eyre::Result<()> {
+    if let Some(file) = opts.file.as_ref() {
+        if opts.url.is_some() {
+            eyre::bail!("specify one of `--url` or `--file` but not both")
+        };
+
+        let mut file = fs::File::open(file)?;
+        let mut bytes = Vec::new();
+        file.read_to_end(&mut bytes)?;
+        codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+        return Ok(())
+    }
+
+    let url = opts.url.unwrap_or_else(|| {
+        "http://localhost:9933"
+            .parse::<Uri>()
+            .expect("default url is valid")
+    });
+    let (_, bytes) = super::metadata::fetch_metadata(&url).await?;
+    codegen(&mut &bytes[..], opts.derives, opts.crate_path)?;
+    Ok(())
+}
+
+fn codegen<I: Input>(
+    encoded: &mut I,
+    raw_derives: Vec<String>,
+    crate_path: Option<String>,
+) -> color_eyre::Result<()> {
+    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(encoded)?;
+    let generator = subxt_codegen::RuntimeGenerator::new(metadata);
+    let item_mod = syn::parse_quote!(
+        pub mod api {}
+    );
+
+    let p = raw_derives
+        .iter()
+        .map(|raw| syn::parse_str(raw))
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let crate_path = crate_path.map(Into::into).unwrap_or_default();
+    let mut derives = DerivesRegistry::new(&crate_path);
+    derives.extend_for_all(p.into_iter());
+
+    let runtime_api = generator.generate_runtime(item_mod, derives, crate_path);
+    println!("{}", runtime_api);
+    Ok(())
+}

cli/src/commands/compatibility.rs

@@ -0,0 +1,136 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use clap::Parser as ClapParser;
+use color_eyre::eyre::{
+    self,
+    WrapErr,
+};
+use frame_metadata::{
+    RuntimeMetadata,
+    RuntimeMetadataPrefixed,
+    RuntimeMetadataV14,
+    META_RESERVED,
+};
+use jsonrpsee::client_transport::ws::Uri;
+use scale::Decode;
+use serde::{
+    Deserialize,
+    Serialize,
+};
+use std::collections::HashMap;
+use subxt_metadata::{
+    get_metadata_hash,
+    get_pallet_hash,
+};
+
+/// Verify metadata compatibility between substrate nodes.
+#[derive(Debug, ClapParser)]
+pub struct Opts {
+    /// Urls of the substrate nodes to verify for metadata compatibility.
+    #[clap(name = "nodes", long, use_delimiter = true, parse(try_from_str))]
+    nodes: Vec<Uri>,
+    /// Check the compatibility of metadata for a particular pallet.
+    ///
+    /// ### Note
+    /// The validation will omit the full metadata check and focus instead on the pallet.
+    #[clap(long, parse(try_from_str))]
+    pallet: Option<String>,
+}
+
+pub async fn run(opts: Opts) -> color_eyre::Result<()> {
+    match opts.pallet {
+        Some(pallet) => {
+            handle_pallet_metadata(opts.nodes.as_slice(), pallet.as_str()).await
+        }
+        None => handle_full_metadata(opts.nodes.as_slice()).await,
+    }
+}
+
+async fn handle_pallet_metadata(nodes: &[Uri], name: &str) -> color_eyre::Result<()> {
+    #[derive(Serialize, Deserialize, Default)]
+    #[serde(rename_all = "camelCase")]
+    struct CompatibilityPallet {
+        pallet_present: HashMap<String, Vec<String>>,
+        pallet_not_found: Vec<String>,
+    }
+
+    let mut compatibility: CompatibilityPallet = Default::default();
+    for node in nodes.iter() {
+        let metadata = fetch_runtime_metadata(node).await?;
+
+        match metadata.pallets.iter().find(|pallet| pallet.name == name) {
+            Some(pallet_metadata) => {
+                let hash = get_pallet_hash(&metadata.types, pallet_metadata);
+                let hex_hash = hex::encode(hash);
+                println!("Node {:?} has pallet metadata hash {:?}", node, hex_hash);
+
+                compatibility
+                    .pallet_present
+                    .entry(hex_hash)
+                    .or_insert_with(Vec::new)
+                    .push(node.to_string());
+            }
+            None => {
+                compatibility.pallet_not_found.push(node.to_string());
+            }
+        }
+    }
+
+    println!(
+        "\nCompatible nodes by pallet\n{}",
+        serde_json::to_string_pretty(&compatibility)
+            .context("Failed to parse compatibility map")?
+    );
+
+    Ok(())
+}
+
+async fn handle_full_metadata(nodes: &[Uri]) -> color_eyre::Result<()> {
+    let mut compatibility_map: HashMap<String, Vec<String>> = HashMap::new();
+    for node in nodes.iter() {
+        let metadata = fetch_runtime_metadata(node).await?;
+        let hash = get_metadata_hash(&metadata);
+        let hex_hash = hex::encode(hash);
+        println!("Node {:?} has metadata hash {:?}", node, hex_hash,);
+
+        compatibility_map
+            .entry(hex_hash)
+            .or_insert_with(Vec::new)
+            .push(node.to_string());
+    }
+
+    println!(
+        "\nCompatible nodes\n{}",
+        serde_json::to_string_pretty(&compatibility_map)
+            .context("Failed to parse compatibility map")?
+    );
+
+    Ok(())
+}
+
+async fn fetch_runtime_metadata(url: &Uri) -> color_eyre::Result<RuntimeMetadataV14> {
+    let (_, bytes) = super::metadata::fetch_metadata(url).await?;
+
+    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
+    if metadata.0 != META_RESERVED {
+        return Err(eyre::eyre!(
+            "Node {:?} has invalid metadata prefix: {:?} expected prefix: {:?}",
+            url,
+            metadata.0,
+            META_RESERVED
+        ))
+    }
+
+    match metadata.1 {
+        RuntimeMetadata::V14(v14) => Ok(v14),
+        _ => {
+            Err(eyre::eyre!(
+                "Node {:?} with unsupported metadata version: {:?}",
+                url,
+                metadata.1
+            ))
+        }
+    }
+}

cli/src/commands/metadata.rs

@@ -0,0 +1,102 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use clap::Parser as ClapParser;
+use color_eyre::eyre;
+use frame_metadata::RuntimeMetadataPrefixed;
+use jsonrpsee::{
+    async_client::ClientBuilder,
+    client_transport::ws::{
+        Uri,
+        WsTransportClientBuilder,
+    },
+    core::{
+        client::ClientT,
+        Error,
+    },
+    http_client::HttpClientBuilder,
+    rpc_params,
+};
+use scale::Decode;
+use std::io::{
+    self,
+    Write,
+};
+
+/// Download metadata from a substrate node, for use with `subxt` codegen.
+#[derive(Debug, ClapParser)]
+pub struct Opts {
+    /// The url of the substrate node to query for metadata.
+    #[clap(
+        name = "url",
+        long,
+        parse(try_from_str),
+        default_value = "http://localhost:9933"
+    )]
+    url: Uri,
+    /// The format of the metadata to display: `json`, `hex` or `bytes`.
+    #[clap(long, short, default_value = "bytes")]
+    format: String,
+}
+
+pub async fn run(opts: Opts) -> color_eyre::Result<()> {
+    let (hex_data, bytes) = fetch_metadata(&opts.url).await?;
+
+    match opts.format.as_str() {
+        "json" => {
+            let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
+            let json = serde_json::to_string_pretty(&metadata)?;
+            println!("{}", json);
+            Ok(())
+        }
+        "hex" => {
+            println!("{}", hex_data);
+            Ok(())
+        }
+        "bytes" => Ok(io::stdout().write_all(&bytes)?),
+        _ => {
+            Err(eyre::eyre!(
+                "Unsupported format `{}`, expected `json`, `hex` or `bytes`",
+                opts.format
+            ))
+        }
+    }
+}
+
+pub async fn fetch_metadata(url: &Uri) -> color_eyre::Result<(String, Vec<u8>)> {
+    let hex_data = match url.scheme_str() {
+        Some("http") => fetch_metadata_http(url).await,
+        Some("ws") | Some("wss") => fetch_metadata_ws(url).await,
+        invalid_scheme => {
+            let scheme = invalid_scheme.unwrap_or("no scheme");
+            Err(eyre::eyre!(format!(
+                "`{}` not supported, expects 'http', 'ws', or 'wss'",
+                scheme
+            )))
+        }
+    }?;
+
+    let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
+
+    Ok((hex_data, bytes))
+}
+
+async fn fetch_metadata_ws(url: &Uri) -> color_eyre::Result<String> {
+    let (sender, receiver) = WsTransportClientBuilder::default()
+        .build(url.to_string().parse::<Uri>().unwrap())
+        .await
+        .map_err(|e| Error::Transport(e.into()))?;
+
+    let client = ClientBuilder::default()
+        .max_notifs_per_subscription(4096)
+        .build_with_tokio(sender, receiver);
+
+    Ok(client.request("state_getMetadata", rpc_params![]).await?)
+}
+
+async fn fetch_metadata_http(url: &Uri) -> color_eyre::Result<String> {
+    let client = HttpClientBuilder::default().build(url.to_string())?;
+
+    Ok(client.request::<String>("state_getMetadata", None).await?)
+}

cli/src/commands/mod.rs

@@ -0,0 +1,7 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+pub mod codegen;
+pub mod compatibility;
+pub mod metadata;