Metadata: Retain a subset of metadata pallets (#879)

* Update cargo.lock to use scale-info v2.4.0 Signed-off-by: Alexandru Vasile <[email protected]> * metadata: Retain only a subset of the metadata Signed-off-by: Alexandru Vasile <[email protected]> * codegen: Generate top level Event Signed-off-by: Alexandru Vasile <[email protected]> * metadata: Only retain DispatchError Signed-off-by: Alexandru Vasile <[email protected]> * metadata: Export just the retain method Signed-off-by: Alexandru Vasile <[email protected]> * cli: Retain pallets Signed-off-by: Alexandru Vasile <[email protected]> * metadata: Do not include extrinsic metadata Signed-off-by: Alexandru Vasile <[email protected]> * retain: Fix clippy Signed-off-by: Alexandru Vasile <[email protected]> * test-runtime: Generate per pallet metadata and rs file Signed-off-by: Alexandru Vasile <[email protected]> * ui-tests: Check per metadata generated files Signed-off-by: Alexandru Vasile <[email protected]> * Revert "test-runtime: Generate per pallet metadata and rs file" This reverts commit 725a2e5. Signed-off-by: Alexandru Vasile <[email protected]> * ui-tests: Adjust path to metadata file Signed-off-by: Alexandru Vasile <[email protected]> * ui-tests: Change drop order to keep `PalletMetadata` around Signed-off-by: Alexandru Vasile <[email protected]> * Update metadata/src/retain.rs Co-authored-by: James Wilson <[email protected]> * Address feedback Signed-off-by: Alexandru Vasile <[email protected]> * retain: Keep extrinsic type Signed-off-by: Alexandru Vasile <[email protected]> * cli: Introduce `MetadataSource` Signed-off-by: Alexandru Vasile <[email protected]> * cli: Use `MetadataSource` helper Signed-off-by: Alexandru Vasile <[email protected]> * cli: Use `FileOrUrl` flatten command argument Signed-off-by: Alexandru Vasile <[email protected]> * retain: Do not include generic type params in retained metadata Signed-off-by: Alexandru Vasile <[email protected]> * Adjust subxt to scale-info v2.5.0 Signed-off-by: Alexandru Vasile <[email protected]> * Update scaleinfo to v2.5.0 Signed-off-by: Alexandru Vasile <[email protected]> * Remove deprecated fn Signed-off-by: Alexandru Vasile <[email protected]> * testing: Fix clippy Signed-off-by: Alexandru Vasile <[email protected]> * benches: Use inner fields of scale info Signed-off-by: Alexandru Vasile <[email protected]> * address nits, and strip RuntimeCall type instead of trying to filter out use of it for better overall wins/clarity * fix UI test * move utils out of commands folder and fix clippy etc * address nits --------- Signed-off-by: Alexandru Vasile <[email protected]> Co-authored-by: James Wilson <[email protected]>
paritytech · Apr 4, 2023 · 8a7c172 · 8a7c172
1 parent c08eb6c
commit 8a7c172
Show file tree

Hide file tree

Showing 14 changed files with 464 additions and 72 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/cli/src/commands/codegen.rs b/cli/src/commands/codegen.rs
@@ -2,10 +2,9 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+use crate::utils::FileOrUrl;
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
-use jsonrpsee::client_transport::ws::Uri;
-use std::{fs, io::Read, path::PathBuf};
 use subxt_codegen::{DerivesRegistry, TypeSubstitutes, TypeSubstitutionError};
 
 /// Generate runtime API client code from metadata.
@@ -15,12 +14,8 @@ use subxt_codegen::{DerivesRegistry, TypeSubstitutes, TypeSubstitutionError};
 /// `subxt codegen | rustfmt --edition=2018 --emit=stdout`
 #[derive(Debug, ClapParser)]
 pub struct Opts {
-    /// The url of the substrate node to query for metadata for codegen.
-    #[clap(name = "url", long, value_parser)]
-    url: Option<Uri>,
-    /// The path to the encoded metadata file.
-    #[clap(short, long, value_parser)]
-    file: Option<PathBuf>,
+    #[command(flatten)]
+    file_or_url: FileOrUrl,
     /// Additional derives
     #[clap(long = "derive")]
     derives: Vec<String>,
@@ -65,23 +60,7 @@ fn substitute_type_parser(src: &str) -> Result<(String, String), String> {
 }
 
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
-    let bytes = if let Some(file) = opts.file.as_ref() {
-        if opts.url.is_some() {
-            eyre::bail!("specify one of `--url` or `--file` but not both")
-        };
-
-        let mut file = fs::File::open(file)?;
-        let mut bytes = Vec::new();
-        file.read_to_end(&mut bytes)?;
-        bytes
-    } else {
-        let url = opts.url.unwrap_or_else(|| {
-            "http://localhost:9933"
-                .parse::<Uri>()
-                .expect("default url is valid")
-        });
-        subxt_codegen::utils::fetch_metadata_bytes(&url).await?
-    };
+    let bytes = opts.file_or_url.fetch().await?;
 
     codegen(
         &bytes,

diff --git a/cli/src/commands/metadata.rs b/cli/src/commands/metadata.rs
@@ -2,47 +2,61 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+use crate::utils::FileOrUrl;
 use clap::Parser as ClapParser;
 use color_eyre::eyre;
-use frame_metadata::RuntimeMetadataPrefixed;
-use jsonrpsee::client_transport::ws::Uri;
-use scale::Decode;
+use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed};
+use scale::{Decode, Encode};
 use std::io::{self, Write};
-use subxt_codegen::utils::fetch_metadata_hex;
+use subxt_metadata::retain_metadata_pallets;
 
 /// Download metadata from a substrate node, for use with `subxt` codegen.
 #[derive(Debug, ClapParser)]
 pub struct Opts {
-    /// The url of the substrate node to query for metadata.
-    #[clap(
-        name = "url",
-        long,
-        value_parser,
-        default_value = "http://localhost:9933"
-    )]
-    url: Uri,
+    #[command(flatten)]
+    file_or_url: FileOrUrl,
     /// The format of the metadata to display: `json`, `hex` or `bytes`.
     #[clap(long, short, default_value = "bytes")]
     format: String,
+    /// Generate a subset of the metadata that contains only the
+    /// types needed to represent the provided pallets.
+    #[clap(long, use_value_delimiter = true, value_parser)]
+    pallets: Option<Vec<String>>,
 }
 
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
-    let hex_data = fetch_metadata_hex(&opts.url).await?;
+    let bytes = opts.file_or_url.fetch().await?;
+    let mut metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
+
+    if let Some(pallets) = opts.pallets {
+        let metadata_v14 = match &mut metadata.1 {
+            RuntimeMetadata::V14(metadata_v14) => metadata_v14,
+            _ => {
+                return Err(eyre::eyre!(
+                    "Unsupported metadata version {:?}, expected V14.",
+                    metadata.1
+                ))
+            }
+        };
+
+        retain_metadata_pallets(metadata_v14, |pallet_name| {
+            pallets.iter().any(|p| &**p == pallet_name)
+        });
+    }
 
     match opts.format.as_str() {
         "json" => {
-            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
-            let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
             let json = serde_json::to_string_pretty(&metadata)?;
             println!("{json}");
             Ok(())
         }
         "hex" => {
+            let hex_data = format!("0x{:?}", hex::encode(metadata.encode()));
             println!("{hex_data}");
             Ok(())
         }
         "bytes" => {
-            let bytes = hex::decode(hex_data.trim_start_matches("0x"))?;
+            let bytes = metadata.encode();
             Ok(io::stdout().write_all(&bytes)?)
         }
         _ => Err(eyre::eyre!(

diff --git a/cli/src/main.rs b/cli/src/main.rs
@@ -5,6 +5,7 @@
 #![deny(unused_crate_dependencies)]
 
 mod commands;
+mod utils;
 use clap::Parser as ClapParser;
 
 /// Subxt utilities for interacting with Substrate based nodes.

diff --git a/cli/src/utils.rs b/cli/src/utils.rs
@@ -0,0 +1,45 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use clap::Args;
+use color_eyre::eyre;
+use std::{fs, io::Read, path::PathBuf};
+use subxt_codegen::utils::Uri;
+
+/// The source of the metadata.
+#[derive(Debug, Args)]
+pub struct FileOrUrl {
+    /// The url of the substrate node to query for metadata for codegen.
+    #[clap(long, value_parser)]
+    url: Option<Uri>,
+    /// The path to the encoded metadata file.
+    #[clap(long, value_parser)]
+    file: Option<PathBuf>,
+}
+
+impl FileOrUrl {
+    /// Fetch the metadata bytes.
+    pub async fn fetch(&self) -> color_eyre::Result<Vec<u8>> {
+        match (&self.file, &self.url) {
+            // Can't provide both --file and --url
+            (Some(_), Some(_)) => {
+                eyre::bail!("specify one of `--url` or `--file` but not both")
+            }
+            // Load from --file path
+            (Some(path), None) => {
+                let mut file = fs::File::open(path)?;
+                let mut bytes = Vec::new();
+                file.read_to_end(&mut bytes)?;
+                Ok(bytes)
+            }
+            // Fetch from --url
+            (None, Some(uri)) => Ok(subxt_codegen::utils::fetch_metadata_bytes(uri).await?),
+            // Default if neither is provided; fetch from local url
+            (None, None) => {
+                let uri = Uri::from_static("http://localhost:9933");
+                Ok(subxt_codegen::utils::fetch_metadata_bytes(&uri).await?)
+            }
+        }
+    }
+}
diff --git a/codegen/src/api/mod.rs b/codegen/src/api/mod.rs
@@ -235,12 +235,13 @@ impl RuntimeGenerator {
     ) -> Result<TokenStream2, CodegenError> {
         let item_mod_attrs = item_mod.attrs.clone();
         let item_mod_ir = ir::ItemMod::try_from(item_mod)?;
+        let default_derives = derives.default_derives();
 
         let type_gen = TypeGenerator::new(
             &self.metadata.types,
             "runtime_types",
             type_substitutes,
-            derives,
+            derives.clone(),
             crate_path.clone(),
             should_gen_docs,
         );
@@ -258,28 +259,6 @@ impl RuntimeGenerator {
             })
             .collect::<Vec<_>>();
 
-        // Get the path to the `Runtime` struct. We assume that the same path contains
-        // RuntimeCall and RuntimeEvent.
-        let runtime_type_id = self.metadata.ty.id;
-        let runtime_path_segments = self
-            .metadata
-            .types
-            .resolve(runtime_type_id)
-            .ok_or(CodegenError::TypeNotFound(runtime_type_id))?
-            .path
-            .namespace()
-            .iter()
-            .map(|part| syn::PathSegment::from(format_ident!("{}", part)));
-        let runtime_path_suffix = syn::Path {
-            leading_colon: None,
-            segments: syn::punctuated::Punctuated::from_iter(runtime_path_segments),
-        };
-        let runtime_path = if runtime_path_suffix.segments.is_empty() {
-            quote!(#types_mod_ident)
-        } else {
-            quote!(#types_mod_ident::#runtime_path_suffix)
-        };
-
         // Pallet names and their length are used to create PALLETS array.
         // The array is used to identify the pallets composing the metadata for
         // validation of just those pallets.
@@ -344,6 +323,26 @@ impl RuntimeGenerator {
             })
             .collect::<Result<Vec<_>, CodegenError>>()?;
 
+        let outer_event_variants = self.metadata.pallets.iter().filter_map(|p| {
+            let variant_name = format_ident!("{}", p.name);
+            let mod_name = format_ident!("{}", p.name.to_string().to_snake_case());
+            let index = proc_macro2::Literal::u8_unsuffixed(p.index);
+
+            p.event.as_ref().map(|_| {
+                quote! {
+                    #[codec(index = #index)]
+                    #variant_name(#mod_name::Event),
+                }
+            })
+        });
+
+        let outer_event = quote! {
+            #default_derives
+            pub enum Event {
+                #( #outer_event_variants )*
+            }
+        };
+
         let root_event_if_arms = self.metadata.pallets.iter().filter_map(|p| {
             let variant_name_str = &p.name;
             let variant_name = format_ident!("{}", variant_name_str);
@@ -402,14 +401,10 @@ impl RuntimeGenerator {
                 // Identify the pallets composing the static metadata by name.
                 pub static PALLETS: [&str; #pallet_names_len] = [ #(#pallet_names,)* ];
 
-                /// The statically generated runtime call type.
-                pub type Call = #runtime_path::RuntimeCall;
-
                 /// The error type returned when there is a runtime issue.
                 pub type DispatchError = #types_mod_ident::sp_runtime::DispatchError;
 
-                // Make the runtime event type easily accessible, and impl RootEvent to help decode into it.
-                pub type Event = #runtime_path::RuntimeEvent;
+                #outer_event
 
                 impl #crate_path::events::RootEvent for Event {
                     fn root_event(pallet_bytes: &[u8], pallet_name: &str, pallet_ty: u32, metadata: &#crate_path::Metadata) -> Result<Self, #crate_path::Error> {

diff --git a/metadata/src/lib.rs b/metadata/src/lib.rs
@@ -2,9 +2,12 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+mod retain;
+
 use frame_metadata::{
     ExtrinsicMetadata, RuntimeMetadataV14, StorageEntryMetadata, StorageEntryType,
 };
+pub use retain::retain_metadata_pallets;
 use scale_info::{form::PortableForm, Field, PortableRegistry, TypeDef, Variant};
 use std::collections::HashSet;