Use scale-typegen as a backend for the codegen

Cargo.toml

@@ -11,19 +11,13 @@ members = [
     "macro",
     "metadata",
     "signer",
-    "subxt"
+    "subxt",
 ]
 
 # We exclude any crates that would depend on non mutually
 # exclusive feature flags and thus can't compile with the
 # workspace:
-exclude = [
-    "testing/wasm-rpc-tests",
-    "testing/wasm-lightclient-tests",
-    "signer/wasm-tests",
-    "examples/wasm-example",
-    "examples/parachain-example"
-]
+exclude = ["testing/wasm-rpc-tests", "testing/wasm-lightclient-tests", "signer/wasm-tests", "examples/wasm-example", "examples/parachain-example"]
 resolver = "2"
 
 [workspace.package]
@@ -80,6 +74,7 @@ trybuild = "1.0.85"
 wabt = "0.10.0"
 wasm-bindgen-test = "0.3.24"
 which = "4.4.2"
+scale-typegen = { git = "https://github.com/paritytech/scale-typegen", branch = "recursive-derives" }
 
 # Light client support:
 smoldot = { version = "0.12.0", default-features = false }

cli/src/commands/codegen.rs

@@ -168,7 +168,8 @@ fn codegen(
             .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?;
         let derive = syn::parse_str(&derive)
             .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?;
-        codegen.add_derives_for_type(ty, std::iter::once(derive));
+        // Note: recursive derives and attributes not supported in the CLI => recursive: false
+        codegen.add_derives_for_type(ty, std::iter::once(derive), false);
     }
 
     // Configure attribtues:
@@ -185,7 +186,8 @@ fn codegen(
             .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?;
         let attribute: OuterAttribute = syn::parse_str(&attr)
             .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?;
-        codegen.add_attributes_for_type(ty, std::iter::once(attribute.0));
+        // Note: recursive derives and attributes not supported in the CLI => recursive: false
+        codegen.add_attributes_for_type(ty, std::iter::once(attribute.0), false);
     }
 
     // Insert type substitutions:
@@ -201,6 +203,10 @@ fn codegen(
         .map_err(|e| eyre!("Cannot decode the provided metadata: {e}"))?;
     let code = codegen
         .generate(metadata)
+        .map_err(|e| {
+            dbg!(&e);
+            e
+        })
         .map_err(|e| eyre!("Cannot generate code: {e}"))?;
 
     writeln!(output, "{code}")?;

codegen/Cargo.toml

@@ -30,11 +30,10 @@ jsonrpsee = { workspace = true, features = ["async-client", "client-ws-transport
 hex = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 thiserror = { workspace = true }
+scale-typegen = { workspace = true }
 
 # Included if "web" feature is enabled, to enable its js feature.
 getrandom = { workspace = true, optional = true }
 
 [dev-dependencies]
-bitvec = { workspace = true }
 scale-info = { workspace = true, features = ["bit-vec"] }
-pretty_assertions = { workspace = true }

codegen/src/api/calls.rs

@@ -3,10 +3,10 @@
 // see LICENSE for license details.
 
 use super::CodegenError;
-use crate::types::{CompositeDefFields, TypeGenerator};
 use heck::{ToSnakeCase as _, ToUpperCamelCase as _};
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
+use scale_typegen::{typegen::ir::type_ir::CompositeIRKind, TypeGenerator};
 use subxt_metadata::PalletMetadata;
 
 /// Generate calls from the provided pallet's metadata. Each call returns a `StaticTxPayload`
@@ -21,48 +21,45 @@ use subxt_metadata::PalletMetadata;
 pub fn generate_calls(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    types_mod_ident: &syn::Ident,
     crate_path: &syn::Path,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     // Early return if the pallet has no calls.
     let Some(call_ty) = pallet.call_ty_id() else {
         return Ok(quote!());
     };
 
-    let mut struct_defs = super::generate_structs_from_variants(
+    let struct_defs = super::generate_structs_from_variants(
         type_gen,
         call_ty,
         |name| name.to_upper_camel_case().into(),
         "Call",
         crate_path,
-        should_gen_docs,
     )?;
     let (call_structs, call_fns): (Vec<_>, Vec<_>) = struct_defs
-        .iter_mut()
-        .map(|(variant_name, struct_def)| {
-            let (call_fn_args, call_args): (Vec<_>, Vec<_>) = match struct_def.fields {
-                CompositeDefFields::Named(ref named_fields) => named_fields
+        .into_iter()
+        .map(|(variant_name, composite)| {
+            let (call_fn_args, call_args): (Vec<_>, Vec<_>) = match &composite.kind {
+                CompositeIRKind::Named(named_fields) => named_fields
                     .iter()
                     .map(|(name, field)| {
                         let fn_arg_type = &field.type_path;
-                        let call_arg = if field.is_boxed() {
+                        let call_arg = if field.is_boxed {
                             quote! { #name: ::std::boxed::Box::new(#name) }
                         } else {
                             quote! { #name }
                         };
                         (quote!( #name: #fn_arg_type ), call_arg)
                     })
                     .unzip(),
-                CompositeDefFields::NoFields => Default::default(),
-                CompositeDefFields::Unnamed(_) => {
+                CompositeIRKind::NoFields => Default::default(),
+                CompositeIRKind::Unnamed(_) => {
                     return Err(CodegenError::InvalidCallVariant(call_ty))
                 }
             };
 
             let pallet_name = pallet.name();
             let call_name = &variant_name;
-            let struct_name = &struct_def.name;
+            let struct_name = &composite.name;
             let Some(call_hash) = pallet.call_hash(call_name) else {
                 return Err(CodegenError::MissingCallMetadata(
                     pallet_name.into(),
@@ -72,8 +69,10 @@ pub fn generate_calls(
             let fn_name = format_ident!("{}", variant_name.to_snake_case());
             // Propagate the documentation just to `TransactionApi` methods, while
             // draining the documentation of inner call structures.
-            let docs = should_gen_docs.then_some(struct_def.docs.take()).flatten();
+            let docs = &composite.docs;
 
+            // this converts the composite into a full struct type. No Type Parameters needed here.
+            let struct_def = type_gen.upcast_composite(&composite);
             // The call structure's documentation was stripped above.
             let call_struct = quote! {
                 #struct_def
@@ -105,12 +104,11 @@ pub fn generate_calls(
         .into_iter()
         .unzip();
 
-    let call_type = type_gen.resolve_type_path(call_ty);
-    let call_ty = type_gen.resolve_type(call_ty);
-    let docs = &call_ty.docs;
-    let docs = should_gen_docs
-        .then_some(quote! { #( #[doc = #docs ] )* })
-        .unwrap_or_default();
+    let call_type = type_gen.resolve_type_path(call_ty)?;
+    let call_ty = type_gen.resolve_type(call_ty)?;
+    let docs = type_gen.docs_from_scale_info(&call_ty.docs);
+
+    let types_mod_ident = type_gen.types_mod_ident();
 
     Ok(quote! {
         #docs

codegen/src/api/constants.rs

@@ -2,10 +2,10 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use crate::types::TypeGenerator;
 use heck::ToSnakeCase as _;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::{format_ident, quote};
+use scale_typegen::TypeGenerator;
 use subxt_metadata::PalletMetadata;
 
 use super::CodegenError;
@@ -36,9 +36,7 @@ use super::CodegenError;
 pub fn generate_constants(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    types_mod_ident: &syn::Ident,
     crate_path: &syn::Path,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     // Early return if the pallet has no constants.
     if pallet.constants().len() == 0 {
@@ -58,9 +56,11 @@ pub fn generate_constants(
                 ));
             };
 
-            let return_ty = type_gen.resolve_type_path(constant.ty());
+            let return_ty = type_gen.resolve_type_path(constant.ty())?;
             let docs = constant.docs();
-            let docs = should_gen_docs
+            let docs = type_gen
+                .settings()
+                .should_gen_docs
                 .then_some(quote! { #( #[doc = #docs ] )* })
                 .unwrap_or_default();
 
@@ -77,6 +77,8 @@ pub fn generate_constants(
         })
         .collect::<Result<Vec<_>, _>>()?;
 
+    let types_mod_ident = type_gen.types_mod_ident();
+
     Ok(quote! {
         pub mod constants {
             use super::#types_mod_ident;

codegen/src/api/custom_values.rs

@@ -2,10 +2,9 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use std::collections::HashSet;
-
-use crate::types::TypeGenerator;
 use heck::ToSnakeCase as _;
+use scale_typegen::TypeGenerator;
+use std::collections::HashSet;
 use subxt_metadata::{CustomValueMetadata, Metadata};
 
 use proc_macro2::TokenStream as TokenStream2;
@@ -60,6 +59,7 @@ fn generate_custom_value_fn(
     let (return_ty, decodable) = if type_is_valid {
         let return_ty = type_gen
             .resolve_type_path(custom_value.type_id())
+            .expect("expected to contain type id")
             .to_token_stream();
         let decodable = quote!(#crate_path::custom_values::Yes);
         (return_ty, decodable)

codegen/src/api/errors.rs

@@ -4,10 +4,9 @@
 
 use proc_macro2::TokenStream as TokenStream2;
 use quote::quote;
+use scale_typegen::TypeGenerator;
 use subxt_metadata::PalletMetadata;
 
-use crate::types::TypeGenerator;
-
 use super::CodegenError;
 
 /// Generate error type alias from the provided pallet metadata.
@@ -20,8 +19,8 @@ pub fn generate_error_type_alias(
         return Ok(quote!());
     };
 
-    let error_type = type_gen.resolve_type_path(error_ty);
-    let error_ty = type_gen.resolve_type(error_ty);
+    let error_type = type_gen.resolve_type_path(error_ty)?;
+    let error_ty = type_gen.resolve_type(error_ty)?;
     let docs = &error_ty.docs;
     let docs = should_gen_docs
         .then_some(quote! { #( #[doc = #docs ] )* })

codegen/src/api/events.rs

@@ -2,9 +2,9 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use crate::types::TypeGenerator;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::quote;
+use scale_typegen::TypeGenerator;
 use subxt_metadata::PalletMetadata;
 
 use super::CodegenError;
@@ -41,9 +41,7 @@ use super::CodegenError;
 pub fn generate_events(
     type_gen: &TypeGenerator,
     pallet: &PalletMetadata,
-    types_mod_ident: &syn::Ident,
     crate_path: &syn::Path,
-    should_gen_docs: bool,
 ) -> Result<TokenStream2, CodegenError> {
     // Early return if the pallet has no events.
     let Some(event_ty) = pallet.event_ty_id() else {
@@ -56,29 +54,33 @@ pub fn generate_events(
         |name| name.into(),
         "Event",
         crate_path,
-        should_gen_docs,
     )?;
 
-    let event_structs = struct_defs.iter().map(|(variant_name, struct_def)| {
+    let event_structs = struct_defs.into_iter().map(|(variant_name, composite)| {
         let pallet_name = pallet.name();
-        let event_struct = &struct_def.name;
+        let event_struct_name = &composite.name;
         let event_name = variant_name;
 
+        let struct_def = type_gen.upcast_composite(&composite);
         quote! {
             #struct_def
 
-            impl #crate_path::events::StaticEvent for #event_struct {
+            impl #crate_path::events::StaticEvent for #event_struct_name {
                 const PALLET: &'static str = #pallet_name;
                 const EVENT: &'static str = #event_name;
             }
         }
     });
-    let event_type = type_gen.resolve_type_path(event_ty);
-    let event_ty = type_gen.resolve_type(event_ty);
+
+    let event_type = type_gen.resolve_type_path(event_ty)?;
+    let event_ty = type_gen.resolve_type(event_ty)?;
     let docs = &event_ty.docs;
-    let docs = should_gen_docs
+    let docs = type_gen
+        .settings()
+        .should_gen_docs
         .then_some(quote! { #( #[doc = #docs ] )* })
         .unwrap_or_default();
+    let types_mod_ident = type_gen.types_mod_ident();
 
     Ok(quote! {
         #docs