From f8f83aec164b9939e32525e8101bcb0d20ca7062 Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 15:45:09 +0200 Subject: [PATCH 1/6] add ModelsCode --- .../autorust/codegen/src/codegen_models.rs | 95 +++++++++++++------ services/autorust/codegen/src/lib.rs | 3 +- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index e75978c0b4..90d4ae1324 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -360,24 +360,50 @@ pub fn all_schemas_resolved(spec: &Spec) -> Result> { Ok(schemas) } -pub fn create_models(cg: &CodeGen) -> Result { - let mut file = TokenStream::new(); +pub enum ModelCode { + Struct(TokenStream), + Enum(StructFieldCode), + VecAlias(TokenStream), + TypeAlias(TypeAliasCode), +} + +impl ToTokens for ModelCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + ModelCode::Struct(struct_code) => struct_code.to_tokens(tokens), + ModelCode::Enum(enum_code) => enum_code.to_tokens(tokens), + ModelCode::VecAlias(vec_alias_code) => vec_alias_code.to_tokens(tokens), + ModelCode::TypeAlias(type_alias_code) => type_alias_code.to_tokens(tokens), + } + } +} - let has_case_workaround = cg.should_workaround_case(); +pub struct ModelsCode { + pub has_case_workaround: bool, + pub models: Vec, +} - file.extend(quote! { - #![allow(non_camel_case_types)] - #![allow(unused_imports)] - use std::str::FromStr; - use serde::{Serialize, Deserialize, Serializer}; - use serde::de::{value, Deserializer, IntoDeserializer}; - }); - if has_case_workaround { - file.extend(quote! { - use azure_core::util::case_insensitive_deserialize; +impl ToTokens for ModelsCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let has_case_workaround = self.has_case_workaround; + let models = &self.models; + tokens.extend(quote! { + #![allow(non_camel_case_types)] + #![allow(unused_imports)] + use std::str::FromStr; + use serde::{Serialize, Deserialize, Serializer}; + use serde::de::{value, Deserializer, IntoDeserializer}; + #(#models)* }); + if has_case_workaround { + tokens.extend(quote! { + use azure_core::util::case_insensitive_deserialize; + }); + } } +} +pub fn create_models(cg: &CodeGen) -> Result { let mut pageable_response_names: HashMap = HashMap::new(); for operation in cg.spec.operations()? { if let Some(pageable) = operation.pageable.as_ref() { @@ -407,46 +433,61 @@ pub fn create_models(cg: &CodeGen) -> Result { // println!("response_names: {:?}", pageable_response_names); + let mut models = Vec::new(); let mut schema_names = IndexMap::new(); for (ref_key, schema) in &all_schemas_resolved(&cg.spec)? { let doc_file = &ref_key.file_path; let schema_name = &ref_key.name; - // println!("schema_name: {}", schema_name); - - // create_response_type() - if let Some(_first_doc_file) = schema_names.insert(schema_name, doc_file) { // eprintln!( // "WARN schema {} already created from {:?}, duplicate from {:?}", // schema_name, _first_doc_file, doc_file // ); } else if schema.is_array() { - file.extend(create_vec_alias(schema)?); + models.push(ModelCode::VecAlias(create_vec_alias(schema)?)); } else if schema.is_local_enum() { let enum_code = create_enum(None, schema, schema_name, false)?; - file.extend(enum_code.into_token_stream()); + models.push(ModelCode::Enum(enum_code)); } else if schema.is_basic_type() { - let (id, value) = create_basic_type_alias(schema_name, schema)?; - file.extend(quote! { pub type #id = #value;}); + let alias = create_basic_type_alias(schema_name, schema)?; + models.push(ModelCode::TypeAlias(alias)); } else { let pageable_name = format!("{}", schema_name.to_camel_case_ident()?); - file.extend(create_struct( + models.push(ModelCode::Struct(create_struct( cg, schema, schema_name, pageable_response_names.get(&pageable_name), HashSet::new(), - )?); + )?)); } } - Ok(file) + Ok(ModelsCode { + has_case_workaround: cg.should_workaround_case(), + models, + }) +} + +pub struct TypeAliasCode { + pub id: Ident, + pub value: TypeNameCode, +} + +impl ToTokens for TypeAliasCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let value = &self.value; + tokens.extend(quote! { + pub type #id = #value; + }); + } } -fn create_basic_type_alias(property_name: &str, property: &SchemaGen) -> Result<(Ident, TypeNameCode)> { +fn create_basic_type_alias(property_name: &str, property: &SchemaGen) -> Result { let id = property_name.to_camel_case_ident()?; let value = TypeNameCode::new(&property.type_name()?)?; - Ok((id, value)) + Ok(TypeAliasCode { id, value }) } // For create_models. Recursively adds schema refs. @@ -906,7 +947,7 @@ fn create_struct( Ok(code) } -struct StructFieldCode { +pub struct StructFieldCode { type_name: TypeNameCode, code: Option, } diff --git a/services/autorust/codegen/src/lib.rs b/services/autorust/codegen/src/lib.rs index 0f6ac88fb2..5d901676e1 100644 --- a/services/autorust/codegen/src/lib.rs +++ b/services/autorust/codegen/src/lib.rs @@ -20,6 +20,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use config_parser::Configuration; pub use error::{Error, ErrorKind, Result, ResultExt}; use proc_macro2::TokenStream; +use quote::ToTokens; use std::io::Write; use std::{ collections::HashSet, @@ -110,7 +111,7 @@ pub fn run<'a>(crate_config: &'a CrateConfig, package_config: &'a PackageConfig) if crate_config.should_run(&Runs::Models) { let models = codegen_models::create_models(&cg)?; let models_path = io::join(&crate_config.output_folder, "models.rs")?; - write_file(models_path, &models, crate_config.print_writing_file())?; + write_file(models_path, &models.to_token_stream(), crate_config.print_writing_file())?; } // create api client from operations From 13e9f5b723f65c98897651af63870fd2419d682a Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 16:10:38 +0200 Subject: [PATCH 2/6] move models to the end --- services/autorust/codegen/src/codegen_models.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index 90d4ae1324..c7b77e9893 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -393,13 +393,15 @@ impl ToTokens for ModelsCode { use std::str::FromStr; use serde::{Serialize, Deserialize, Serializer}; use serde::de::{value, Deserializer, IntoDeserializer}; - #(#models)* }); if has_case_workaround { tokens.extend(quote! { use azure_core::util::case_insensitive_deserialize; }); } + tokens.extend(quote! { + #(#models)* + }); } } From 13c680915cce24431132b1de1488f4aae95141f5 Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 17:02:50 +0200 Subject: [PATCH 3/6] add VecAliasCode --- .../autorust/codegen/src/codegen_models.rs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index c7b77e9893..4c0522d734 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -363,7 +363,7 @@ pub fn all_schemas_resolved(spec: &Spec) -> Result> { pub enum ModelCode { Struct(TokenStream), Enum(StructFieldCode), - VecAlias(TokenStream), + VecAlias(VecAliasCode), TypeAlias(TypeAliasCode), } @@ -667,11 +667,26 @@ fn create_enum( }) } -fn create_vec_alias(schema: &SchemaGen) -> Result { +pub struct VecAliasCode { + pub id: Ident, + pub value: TypeNameCode, +} + +impl ToTokens for VecAliasCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let id = &self.id; + let value = &self.value; + tokens.extend(quote! { + pub type #id = Vec<#value>; + }); + } +} + +fn create_vec_alias(schema: &SchemaGen) -> Result { let items = schema.array_items()?; - let typ = schema.name()?.to_camel_case_ident()?; - let items_typ = TypeNameCode::new(&get_type_name_for_schema_ref(items)?)?; - Ok(quote! { pub type #typ = Vec<#items_typ>; }) + let id = schema.name()?.to_camel_case_ident()?; + let value = TypeNameCode::new(&get_type_name_for_schema_ref(items)?)?; + Ok(VecAliasCode { id, value }) } fn create_struct( From 87bd3306ac474735b1bccfc214c3ed2049a9ef2f Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 18:01:31 +0200 Subject: [PATCH 4/6] start StructCode --- .../autorust/codegen/src/codegen_models.rs | 134 ++++++++++++------ 1 file changed, 87 insertions(+), 47 deletions(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index 4c0522d734..c5507f159c 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -361,7 +361,7 @@ pub fn all_schemas_resolved(spec: &Spec) -> Result> { } pub enum ModelCode { - Struct(TokenStream), + Struct(StructCode), Enum(StructFieldCode), VecAlias(VecAliasCode), TypeAlias(TypeAliasCode), @@ -689,14 +689,83 @@ fn create_vec_alias(schema: &SchemaGen) -> Result { Ok(VecAliasCode { id, value }) } +pub struct StructCode { + doc_comment: TokenStream, + struct_name_code: Ident, + default_code: TokenStream, + props: TokenStream, + continuable: TokenStream, + implement_default: bool, + new_fn_params: Vec, + new_fn_body: TokenStream, + mod_code: TokenStream, + ns: Ident, +} + +impl ToTokens for StructCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let StructCode { + doc_comment, + struct_name_code, + default_code, + props, + continuable, + implement_default, + new_fn_params, + new_fn_body, + mod_code, + ns, + } = self; + + let struct_code = quote! { + #doc_comment + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] + #default_code + pub struct #struct_name_code { + #props + } + #continuable + }; + tokens.extend(struct_code); + + tokens.extend(if *implement_default { + quote! { + impl #struct_name_code { + pub fn new() -> Self { + Self::default() + } + } + } + } else { + quote! { + impl #struct_name_code { + pub fn new(#(#new_fn_params),*) -> Self { + Self { + #new_fn_body + } + } + } + } + }); + + if !mod_code.is_empty() { + tokens.extend(quote! { + pub mod #ns { + use super::*; + #mod_code + } + }); + } + } +} + fn create_struct( cg: &CodeGen, schema: &SchemaGen, struct_name: &str, pageable: Option<&MsPageable>, mut needs_boxing: HashSet, -) -> Result { - let mut code = TokenStream::new(); +) -> Result { let mut mod_code = TokenStream::new(); let mut props = TokenStream::new(); let mut new_fn_params: Vec = Vec::new(); @@ -921,47 +990,18 @@ fn create_struct( } } - let struct_code = quote! { - #doc_comment - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] - #default_code - pub struct #struct_name_code { - #props - } - #continuable - }; - code.extend(struct_code); - - code.extend(if schema.implement_default() { - quote! { - impl #struct_name_code { - pub fn new() -> Self { - Self::default() - } - } - } - } else { - quote! { - impl #struct_name_code { - pub fn new(#(#new_fn_params),*) -> Self { - Self { - #new_fn_body - } - } - } - } - }); - - if !mod_code.is_empty() { - code.extend(quote! { - pub mod #ns { - use super::*; - #mod_code - } - }); - } - - Ok(code) + Ok(StructCode { + doc_comment, + struct_name_code, + default_code, + props, + continuable, + implement_default: schema.implement_default(), + new_fn_params, + new_fn_body, + mod_code, + ns, + }) } pub struct StructFieldCode { @@ -978,7 +1018,7 @@ impl ToTokens for StructFieldCode { } enum TypeCode { - Struct(TokenStream), + Struct(StructCode), Enum(TokenStream), XmlWrapped(XmlWrappedCode), } @@ -986,8 +1026,8 @@ enum TypeCode { impl ToTokens for TypeCode { fn to_tokens(&self, tokens: &mut TokenStream) { match self { - TypeCode::Struct(code) => tokens.extend(code.clone()), - TypeCode::Enum(code) => tokens.extend(code.clone()), + TypeCode::Struct(code) => code.to_tokens(tokens), + TypeCode::Enum(code) => code.to_tokens(tokens), TypeCode::XmlWrapped(code) => code.to_tokens(tokens), } } From 6fbf3d7acd2a4154671210ff057cb48653048b84 Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 19:39:56 +0200 Subject: [PATCH 5/6] add DocCommentCode --- .../autorust/codegen/src/codegen_models.rs | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index c5507f159c..f9e5243b24 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -519,12 +519,7 @@ fn create_enum( for enum_value in &enum_values { let value = &enum_value.value; let nm = value.to_camel_case_ident()?; - let doc_comment = match &enum_value.description { - Some(description) => { - quote! { #[doc = #description] } - } - None => quote! {}, - }; + let doc_comment = DocCommentCode::new(&enum_value.description); let lower = value.to_lowercase(); let rename = if &nm.to_string() == value { quote! {} @@ -642,12 +637,7 @@ fn create_enum( quote! {} }; - let doc_comment = match &property.schema.common.description { - Some(description) => { - quote! { #[doc = #description] } - } - None => quote! {}, - }; + let doc_comment = DocCommentCode::new(&property.schema.common.description); let code = quote! { #doc_comment @@ -690,7 +680,7 @@ fn create_vec_alias(schema: &SchemaGen) -> Result { } pub struct StructCode { - doc_comment: TokenStream, + doc_comment: DocCommentCode, struct_name_code: Ident, default_code: TokenStream, props: TokenStream, @@ -891,10 +881,7 @@ fn create_struct( } type_name = type_name.boxed(boxed); - let doc_comment = match &property.schema.schema.common.description { - Some(description) => quote! { #[doc = #description] }, - None => quote! {}, - }; + let doc_comment = DocCommentCode::new(&property.schema.schema.common.description); props.extend(quote! { #doc_comment @@ -923,10 +910,7 @@ fn create_struct( quote! {} }; - let doc_comment = match &schema.schema.common.description { - Some(description) => quote! { #[doc = #description] }, - None => quote! {}, - }; + let doc_comment = DocCommentCode::new(&schema.schema.common.description); let mut continuable = quote! {}; if let Some(pageable) = pageable { @@ -1004,6 +988,28 @@ fn create_struct( }) } +pub struct DocCommentCode { + description: Option, +} + +impl DocCommentCode { + pub fn new(description: &Option) -> Self { + Self { + description: description.clone(), + } + } +} + +impl ToTokens for DocCommentCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Some(description) = &self.description { + tokens.extend(quote! { + #[doc = #description] + }); + } + } +} + pub struct StructFieldCode { type_name: TypeNameCode, code: Option, From f3de311408ee6082e40c4ea3ff72856ca894d1ce Mon Sep 17 00:00:00 2001 From: Cameron Taggart Date: Tue, 19 Sep 2023 23:37:34 +0200 Subject: [PATCH 6/6] add DocCommentCode, SerdeCode, StructPropCode --- .../autorust/codegen/src/codegen_models.rs | 112 ++++++++++++++---- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/services/autorust/codegen/src/codegen_models.rs b/services/autorust/codegen/src/codegen_models.rs index f9e5243b24..3ad7b31795 100644 --- a/services/autorust/codegen/src/codegen_models.rs +++ b/services/autorust/codegen/src/codegen_models.rs @@ -519,7 +519,7 @@ fn create_enum( for enum_value in &enum_values { let value = &enum_value.value; let nm = value.to_camel_case_ident()?; - let doc_comment = DocCommentCode::new(&enum_value.description); + let doc_comment = DocCommentCode::from(&enum_value.description); let lower = value.to_lowercase(); let rename = if &nm.to_string() == value { quote! {} @@ -637,7 +637,7 @@ fn create_enum( quote! {} }; - let doc_comment = DocCommentCode::new(&property.schema.common.description); + let doc_comment = DocCommentCode::from(&property.schema.common.description); let code = quote! { #doc_comment @@ -683,7 +683,7 @@ pub struct StructCode { doc_comment: DocCommentCode, struct_name_code: Ident, default_code: TokenStream, - props: TokenStream, + props: Vec, continuable: TokenStream, implement_default: bool, new_fn_params: Vec, @@ -712,7 +712,7 @@ impl ToTokens for StructCode { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #default_code pub struct #struct_name_code { - #props + #(#props)* } #continuable }; @@ -757,7 +757,7 @@ fn create_struct( mut needs_boxing: HashSet, ) -> Result { let mut mod_code = TokenStream::new(); - let mut props = TokenStream::new(); + let mut props = Vec::new(); let mut new_fn_params: Vec = Vec::new(); let mut new_fn_body = TokenStream::new(); let ns = struct_name.to_snake_case_ident()?; @@ -771,9 +771,11 @@ fn create_struct( let schema_name = schema.name()?; let type_name = schema_name.to_camel_case_ident()?; let field_name = schema_name.to_snake_case_ident()?; - props.extend(quote! { - #[serde(flatten)] - pub #field_name: #type_name, + props.push(StructPropCode { + doc_comments: Vec::new(), + serde: SerdeCode::flatten(), + field_name: field_name.clone(), + field_type: type_name.clone().into(), }); if schema.implement_default() { new_fn_body.extend(quote! { #field_name: #type_name::default(), }); @@ -812,9 +814,10 @@ fn create_struct( needs_boxing.clone(), )?; mod_code.extend(field_code.into_token_stream()); + let mut doc_comments = Vec::new(); // uncomment the next two lines to help identify entries that need boxed // let prop_nm_str = format!("{} , {} , {}", prop_nm.file_path, prop_nm.schema_name, property_name); - // props.extend(quote! { #[doc = #prop_nm_str ]}); + // doc_comments.push(DocCommentCode::from(&Some(prop_nm_str))); let mut boxed = false; if needs_boxing.contains(&type_name.to_string().to_camel_case_ident()?.to_string()) { @@ -869,11 +872,7 @@ fn create_struct( serde_attrs.push(quote! { with = "azure_core::xml::text_content"}); } } - let serde = if !serde_attrs.is_empty() { - quote! { #[serde(#(#serde_attrs),*)] } - } else { - quote! {} - }; + let serde = SerdeCode::new(serde_attrs); // see if a field should be wrapped in a Box if cg.should_box_property(prop_nm) { @@ -881,12 +880,13 @@ fn create_struct( } type_name = type_name.boxed(boxed); - let doc_comment = DocCommentCode::new(&property.schema.schema.common.description); + doc_comments.push(DocCommentCode::from(&property.schema.schema.common.description)); - props.extend(quote! { - #doc_comment - #serde - pub #field_name: #type_name, + props.push(StructPropCode { + doc_comments, + serde, + field_name: field_name.clone(), + field_type: type_name.clone(), }); if is_required { @@ -910,7 +910,7 @@ fn create_struct( quote! {} }; - let doc_comment = DocCommentCode::new(&schema.schema.common.description); + let doc_comment = DocCommentCode::from(&schema.schema.common.description); let mut continuable = quote! {}; if let Some(pageable) = pageable { @@ -988,12 +988,80 @@ fn create_struct( }) } +pub struct StructPropCode { + pub doc_comments: Vec, + pub serde: SerdeCode, + pub field_name: Ident, + pub field_type: TypeNameCode, +} + +impl StructPropCode { + pub fn new(field_name: Ident, field_type: TypeNameCode) -> Self { + Self { + doc_comments: Vec::new(), + serde: SerdeCode::default(), + field_name, + field_type, + } + } +} + +impl ToTokens for StructPropCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let doc_comments = &self.doc_comments; + let serde = &self.serde; + let field_name = &self.field_name; + let field_type = &self.field_type; + tokens.extend(quote! { + #(#doc_comments)* + #serde + pub #field_name: #field_type, + }); + } +} + +#[derive(Default)] +pub struct SerdeCode { + pub attributes: Vec, +} + +impl SerdeCode { + pub fn new(attributes: Vec) -> Self { + Self { attributes } + } + pub fn flatten() -> Self { + Self { + attributes: vec![quote! { flatten }], + } + } +} + +impl ToTokens for SerdeCode { + fn to_tokens(&self, tokens: &mut TokenStream) { + let attributes = &self.attributes; + if !attributes.is_empty() { + tokens.extend(quote! { + #[serde(#(#attributes),*)] + }); + } + } +} + +#[derive(Default)] pub struct DocCommentCode { description: Option, } -impl DocCommentCode { - pub fn new(description: &Option) -> Self { +impl From<&str> for DocCommentCode { + fn from(description: &str) -> Self { + Self { + description: Some(description.to_string()), + } + } +} + +impl From<&Option> for DocCommentCode { + fn from(description: &Option) -> Self { Self { description: description.clone(), }