From d1f36fc71a51a8155ea5d3fba085257391907c43 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 01:09:55 +0800 Subject: [PATCH 01/14] parse deprecated note from config --- dev/Cargo.lock | 47 +++++++++++++++++++ dev/Cargo.toml | 5 ++- dev/src/generate/binding_python.rs | 3 +- dev/src/generate/mod.rs | 3 +- dev/src/generate/parser.rs | 72 ++++++++++++++++++++++++++---- 5 files changed, 119 insertions(+), 11 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index bbc6ca74ea1e..d1a8e49471ac 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -118,6 +118,21 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -159,6 +174,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "log" version = "0.4.22" @@ -177,9 +201,12 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", + "enquote", "env_logger", + "itertools", "log", "pretty_assertions", + "proc-macro2", "syn", ] @@ -257,6 +284,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 7674646a94a8..31120d77f674 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -30,9 +30,12 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } +enquote = "1.1.0" env_logger = "0.11.6" +itertools = "0.13.0" log = "0.4.22" -syn = { version = "2.0.91", features = ["visit","full","extra-traits"] } +proc-macro2 = { version = "1.0.91", features = ["span-locations"] } +syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 7f3e3bdef83f..c6bef062034e 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,8 +17,9 @@ use crate::generate::parser::Services; use anyhow::Result; +use std::path::PathBuf; -pub fn generate(services: &Services) -> Result<()> { +pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> { println!("{:?}", services); Ok(()) diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 743aa3b1c483..71ec44429998 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -25,10 +25,11 @@ use std::path::PathBuf; pub fn run(language: &str) -> Result<()> { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let services_path = manifest_dir.join("../core/src/services").canonicalize()?; + let project_root = manifest_dir.join("..").canonicalize()?; let services = parser::parse(&services_path.to_string_lossy())?; match language { - "python" | "py" => binding_python::generate(&services), + "python" | "py" => binding_python::generate(project_root, &services), _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index e8c3b6f03551..6a265f2f2c14 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -17,23 +17,23 @@ use anyhow::Result; use anyhow::{anyhow, Context}; +use itertools::Itertools; use log::debug; -use std::collections::hash_map; use std::collections::HashMap; -use std::fs; use std::fs::read_dir; use std::str::FromStr; -use syn::{Field, GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath}; +use std::{fs, vec}; +use syn::{Field, GenericArgument, Item, PathArguments, Type, TypePath}; #[derive(Debug, Clone)] pub struct Services(HashMap); impl IntoIterator for Services { type Item = (String, Service); - type IntoIter = hash_map::IntoIter; + type IntoIter = vec::IntoIter<(String, Service)>; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + self.0.into_iter().sorted_by_key(|x| x.0.clone()) } } @@ -53,6 +53,8 @@ pub struct Config { pub value: ConfigType, /// If given config is optional or not. pub optional: bool, + /// if this field is deprecated, a deprecated message will be provided. + pub deprecated: Option, /// The comments for this config. /// /// All white spaces and extra new lines will be trimmed. @@ -181,7 +183,7 @@ impl ServiceParser { let mut config = Vec::with_capacity(config_struct.fields.len()); for field in config_struct.fields { - let field = Self::parse_field(field)?; + let field = self.parse_field(field)?; config.push(field); } @@ -190,12 +192,14 @@ impl ServiceParser { } /// TODO: Add comment parse support. - fn parse_field(field: Field) -> Result { + fn parse_field(&self, field: Field) -> Result { let name = field .ident .clone() .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; + let deprecated = self.deprecated_note(&field); + let (cfg_type, optional) = match &field.ty { Type::Path(TypePath { path, .. }) => { let segment = path @@ -226,6 +230,7 @@ impl ServiceParser { }; let typ = type_name.as_str().parse()?; + let optional = optional || typ == ConfigType::Bool; (typ, optional) } @@ -236,9 +241,44 @@ impl ServiceParser { name: name.to_string(), value: cfg_type, optional, + deprecated, comments: "".to_string(), }) } + + fn deprecated_note(&self, field: &Field) -> Option { + for attr in &field.attrs { + if !attr.path().is_ident("deprecated") { + continue; + } + + let meta_list = match &attr.meta { + syn::Meta::List(meta_list) => meta_list, + _ => continue, + }; + + let tokens = Vec::from_iter(meta_list.tokens.clone().into_iter()); + for (index, token) in tokens.iter().enumerate() { + let ident = match token { + proc_macro2::TokenTree::Ident(ident) => ident, + _ => { + continue; + } + }; + + if ident == "note" { + return tokens + .get(index + 2) + .expect("deprecated attribute missing note") + .span() + .source_text() + .map(|s| enquote::unquote(s.as_str()).expect("should unquote string")); + } + } + } + + return None; + } } #[cfg(test)] @@ -246,6 +286,7 @@ mod tests { use super::*; use pretty_assertions::assert_eq; use std::path::PathBuf; + use syn::ItemStruct; #[test] fn test_parse_field() { @@ -256,6 +297,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: true, + deprecated: None, comments: "".to_string(), }, ), @@ -265,6 +307,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: false, + deprecated: None, comments: "".to_string(), }, ), @@ -495,12 +538,25 @@ impl Debug for S3Config { let service = parser.parse().unwrap(); assert_eq!(service.config.len(), 26); + assert_eq!( + service.config[21], + Config { + name: "batch_max_operations".to_string(), + value: ConfigType::Usize, + optional: true, + deprecated: Some( + "Please use `delete_max_size` instead of `batch_max_operations`".into() + ), + comments: "".to_string(), + }, + ); assert_eq!( service.config[25], Config { name: "disable_write_with_if_match".to_string(), value: ConfigType::Bool, - optional: false, + optional: true, + deprecated: None, comments: "".to_string(), }, ); From deb8eb668b3b2f57289c646027b738c79809310e Mon Sep 17 00:00:00 2001 From: Trim21 Date: Wed, 25 Dec 2024 19:16:47 +0800 Subject: [PATCH 02/14] add option deprecated note --- dev/Cargo.toml | 2 +- dev/src/generate/parser.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 31120d77f674..eb0397756f27 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -35,7 +35,7 @@ env_logger = "0.11.6" itertools = "0.13.0" log = "0.4.22" proc-macro2 = { version = "1.0.91", features = ["span-locations"] } -syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } +syn = { version = "2.0.91", features = ["visit", "full", "extra-traits",'printing', 'parsing'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 6a265f2f2c14..68fa46bbc974 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use anyhow::Result; use anyhow::{anyhow, Context}; use itertools::Itertools; use log::debug; @@ -237,6 +236,8 @@ impl ServiceParser { v => return Err(anyhow!("unsupported config type {v:?}")), }; + deprecated = deprecated.map(|s| s[1..s.len() - 1].into()); + Ok(Config { name: name.to_string(), value: cfg_type, @@ -284,6 +285,7 @@ impl ServiceParser { #[cfg(test)] mod tests { use super::*; + use pretty_assertions::assert_eq; use std::path::PathBuf; use syn::ItemStruct; From 412d013538279d2447e633eea1dc60a1cb7e0b23 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 26 Dec 2024 01:16:38 +0800 Subject: [PATCH 03/14] gen --- bindings/python/Cargo.toml | 7 +- bindings/python/python/opendal/__base.pyi | 720 ++++++++++++++++++++ bindings/python/python/opendal/__init__.pyi | 9 +- dev/Cargo.toml | 2 +- dev/src/generate/binding_python.rs | 80 ++- dev/src/generate/parser.rs | 2 - dev/templates/python | 44 ++ 7 files changed, 851 insertions(+), 13 deletions(-) create mode 100644 bindings/python/python/opendal/__base.pyi create mode 100644 dev/templates/python diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 6cda36ddfb29..bf57397945e1 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -45,6 +45,9 @@ default = [ "services-webhdfs", ] +# NOTE: this is the feature we used to build pypi wheels. +# When enable or disable some features, +# also need to update dev/src/generate/binding_python.rs `enabled_service` to match it. services-all = [ "default", "services-aliyun-drive", @@ -161,9 +164,7 @@ bytes = "1.5.0" dict_derive = "0.6.0" futures = "0.3.28" # this crate won't be published, we always use the local version -opendal = { version = ">=0", path = "../../core", features = [ - "layers-blocking", -] } +opendal = { version = ">=0", path = "../../core", features = ["layers-blocking"] } pyo3 = { version = "0.23.3", features = ["generate-import-lib"] } pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] } tokio = "1" diff --git a/bindings/python/python/opendal/__base.pyi b/bindings/python/python/opendal/__base.pyi new file mode 100644 index 000000000000..d18fbed11cdb --- /dev/null +++ b/bindings/python/python/opendal/__base.pyi @@ -0,0 +1,720 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """ + + @overload + def __init__( + self, + scheme: Literal["aliyun_drive"], + *, + drive_type: str, + root: str = ..., + access_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + refresh_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["alluxio"], + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["atomicserver"], + *, + root: str = ..., + endpoint: str = ..., + private_key: str = ..., + public_key: str = ..., + parent_resource_id: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azblob"], + *, + container: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + encryption_key: str = ..., + encryption_key_sha256: str = ..., + encryption_algorithm: str = ..., + sas_token: str = ..., + batch_max_operations: _int = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azdls"], + *, + filesystem: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azfile"], + *, + share_name: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + sas_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["b2"], + *, + bucket: str, + bucket_id: str, + root: str = ..., + application_key_id: str = ..., + application_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cacache"], + *, + datadir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["chainsafe"], + *, + bucket_id: str, + root: str = ..., + api_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cloudflare_kv"], + *, + token: str = ..., + account_id: str = ..., + namespace_id: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["compfs"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cos"], + *, + root: str = ..., + endpoint: str = ..., + secret_id: str = ..., + secret_key: str = ..., + bucket: str = ..., + disable_config_load: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["d1"], + *, + token: str = ..., + account_id: str = ..., + database_id: str = ..., + root: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dashmap"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dbfs"], + *, + root: str = ..., + endpoint: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dropbox"], + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["fs"], + *, + root: str = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gcs"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + scope: str = ..., + service_account: str = ..., + credential: str = ..., + credential_path: str = ..., + predefined_acl: str = ..., + default_storage_class: str = ..., + allow_anonymous: _bool = ..., + disable_vm_metadata: _bool = ..., + disable_config_load: _bool = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gdrive"], + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ghac"], + *, + root: str = ..., + version: str = ..., + endpoint: str = ..., + runtime_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["github"], + *, + owner: str, + repo: str, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gridfs"], + *, + connection_string: str = ..., + database: str = ..., + bucket: str = ..., + chunk_size: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["hdfs_native"], + *, + root: str = ..., + url: str = ..., + enable_append: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["http"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["huggingface"], + *, + repo_type: str = ..., + repo_id: str = ..., + revision: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["icloud"], + *, + root: str = ..., + apple_id: str = ..., + password: str = ..., + trust_token: str = ..., + ds_web_auth_token: str = ..., + is_china_mainland: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipfs"], + *, + endpoint: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipmfs"], + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["koofr"], + *, + endpoint: str, + email: str, + root: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["lakefs"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + repository: str = ..., + branch: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["libsql"], + *, + connection_string: str = ..., + auth_token: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memcached"], + *, + endpoint: str = ..., + root: str = ..., + username: str = ..., + password: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memory"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mini_moka"], + *, + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["moka"], + *, + name: str = ..., + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + num_segments: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mongodb"], + *, + connection_string: str = ..., + database: str = ..., + collection: str = ..., + root: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["monoiofs"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mysql"], + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["nebula_graph"], + *, + host: str = ..., + port: _int = ..., + username: str = ..., + password: str = ..., + space: str = ..., + tag: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["obs"], + *, + root: str = ..., + endpoint: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + bucket: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["onedrive"], + *, + access_token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["oss"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + presign_endpoint: str = ..., + server_side_encryption: str = ..., + server_side_encryption_key_id: str = ..., + allow_anonymous: _bool = ..., + access_key_id: str = ..., + access_key_secret: str = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + role_arn: str = ..., + role_session_name: str = ..., + oidc_provider_arn: str = ..., + oidc_token_file: str = ..., + sts_endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["pcloud"], + *, + endpoint: str, + root: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["persy"], + *, + datafile: str = ..., + segment: str = ..., + index: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["postgresql"], + *, + root: str = ..., + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redb"], + *, + datadir: str = ..., + root: str = ..., + table: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redis"], + *, + db: _int, + endpoint: str = ..., + cluster_endpoints: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["s3"], + *, + bucket: str, + root: str = ..., + enable_versioning: _bool = ..., + endpoint: str = ..., + region: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + session_token: str = ..., + role_arn: str = ..., + external_id: str = ..., + role_session_name: str = ..., + disable_config_load: _bool = ..., + disable_ec2_metadata: _bool = ..., + allow_anonymous: _bool = ..., + server_side_encryption: str = ..., + server_side_encryption_aws_kms_key_id: str = ..., + server_side_encryption_customer_algorithm: str = ..., + server_side_encryption_customer_key: str = ..., + server_side_encryption_customer_key_md5: str = ..., + default_storage_class: str = ..., + enable_virtual_host_style: _bool = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + disable_stat_with_override: _bool = ..., + checksum_algorithm: str = ..., + disable_write_with_if_match: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["seafile"], + *, + repo_name: str, + root: str = ..., + endpoint: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sftp"], + *, + endpoint: str = ..., + root: str = ..., + user: str = ..., + key: str = ..., + known_hosts_strategy: str = ..., + enable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sled"], + *, + datadir: str = ..., + root: str = ..., + tree: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sqlite"], + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["supabase"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["surrealdb"], + *, + connection_string: str = ..., + username: str = ..., + password: str = ..., + namespace: str = ..., + database: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["swift"], + *, + endpoint: str = ..., + container: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["upyun"], + *, + bucket: str, + root: str = ..., + operator: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_artifacts"], + *, + access_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_blob"], + *, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webdav"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + disable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webhdfs"], + *, + root: str = ..., + endpoint: str = ..., + delegation: str = ..., + disable_list_batch: _bool = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["yandex_disk"], + *, + access_token: str, + root: str = ..., + ) -> None: ... + @overload + def __init__(self, scheme: str, **kwargs: str) -> None: ... diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi index 6d6645a60bbc..663cbc677c20 100644 --- a/bindings/python/python/opendal/__init__.pyi +++ b/bindings/python/python/opendal/__init__.pyi @@ -15,16 +15,16 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, AsyncIterable, Iterable, Optional, final, Union, Type +from typing import AsyncIterable, Iterable, Optional, final, Union, Type from types import TracebackType from opendal import exceptions as exceptions from opendal import layers as layers from opendal.layers import Layer +from opendal.__base import _Base @final -class Operator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class Operator(_Base): def layer(self, layer: Layer) -> "Operator": ... def open(self, path: str, mode: str) -> File: ... def read(self, path: str) -> bytes: ... @@ -51,8 +51,7 @@ class Operator: def to_async_operator(self) -> AsyncOperator: ... @final -class AsyncOperator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class AsyncOperator(_Base): def layer(self, layer: Layer) -> "AsyncOperator": ... async def open(self, path: str, mode: str) -> AsyncFile: ... async def read(self, path: str) -> bytes: ... diff --git a/dev/Cargo.toml b/dev/Cargo.toml index eb0397756f27..31120d77f674 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -35,7 +35,7 @@ env_logger = "0.11.6" itertools = "0.13.0" log = "0.4.22" proc-macro2 = { version = "1.0.91", features = ["span-locations"] } -syn = { version = "2.0.91", features = ["visit", "full", "extra-traits",'printing', 'parsing'] } +syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index c6bef062034e..b03471715c60 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,10 +17,86 @@ use crate::generate::parser::Services; use anyhow::Result; +use itertools::Itertools; +use std::fs; use std::path::PathBuf; -pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> { - println!("{:?}", services); +use super::parser::ConfigType; + +fn enabled_service(srv: &str) -> bool { + match srv { + // not enabled in bindings/python/Cargo.toml + "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, + _ => true, + } +} + +pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { + let mut s = fs::read_to_string(project_root.join("dev/templates/python")) + .expect("failed to read python template file"); + + for (srv, config) in services.clone().into_iter() { + if !enabled_service(srv.as_str()) { + continue; + } + + s.push_str("\n @overload\n"); + s.push_str(" def __init__(self,\n"); + s.push_str(format!("scheme: Literal[\"{}\"],", srv).as_str()); + s.push_str("\n*,\n"); + + for (_, f) in config + .config + .into_iter() + .enumerate() + .sorted_by_key(|(i, x)| (x.optional, *i)) + { + if let Some(deprecated) = f.deprecated { + s.push_str("# deprecated: "); + s.push_str(deprecated.as_str()); + s.push('\n'); + } + + s.push_str(&f.name); + s.push_str(": "); + match f.value { + ConfigType::Bool => { + s.push_str("_bool"); + } + ConfigType::Duration => { + s.push_str("_duration"); + } + ConfigType::I64 + | ConfigType::Usize + | ConfigType::U64 + | ConfigType::U32 + | ConfigType::U16 => { + s.push_str("_int"); + } + ConfigType::Vec => { + s.push_str("_strings"); + } + ConfigType::String => { + s.push_str("str"); + } + } + if f.optional { + s.push_str(" = ...,\n"); + } else { + s.push_str(",\n"); + } + } + + s.push_str(")->None:...\n"); + } + + s.push_str(" @overload\n def __init__(self, scheme:str, **kwargs: str) -> None: ...\n"); + + fs::write( + project_root.join("bindings/python/python/opendal/__base.pyi"), + s, + ) + .expect("failed to write result to file"); Ok(()) } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 68fa46bbc974..8e49a08206fe 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -236,8 +236,6 @@ impl ServiceParser { v => return Err(anyhow!("unsupported config type {v:?}")), }; - deprecated = deprecated.map(|s| s[1..s.len() - 1].into()); - Ok(Config { name: name.to_string(), value: cfg_type, diff --git a/dev/templates/python b/dev/templates/python new file mode 100644 index 000000000000..f0e662fa0ab9 --- /dev/null +++ b/dev/templates/python @@ -0,0 +1,44 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """ From 5d309aee2571ac686cedcfcabdcff2d6cc077d42 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 26 Dec 2024 08:45:42 +0800 Subject: [PATCH 04/14] add a TODO --- dev/src/generate/binding_python.rs | 1 + dev/src/generate/parser.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index b03471715c60..0a712f8dd3d0 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -23,6 +23,7 @@ use std::path::PathBuf; use super::parser::ConfigType; +/// TODO: add a common utils to parse enabled features from cargo.toml fn enabled_service(srv: &str) -> bool { match srv { // not enabled in bindings/python/Cargo.toml diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 8e49a08206fe..6a265f2f2c14 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use anyhow::Result; use anyhow::{anyhow, Context}; use itertools::Itertools; use log::debug; @@ -283,7 +284,6 @@ impl ServiceParser { #[cfg(test)] mod tests { use super::*; - use pretty_assertions::assert_eq; use std::path::PathBuf; use syn::ItemStruct; From 66cabb9ffd0321a521997a54f7732549ebcca895 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 03:26:54 +0800 Subject: [PATCH 05/14] run ruff format --- dev/src/generate/binding_python.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 0a712f8dd3d0..ad1ffa0d47ec 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -20,6 +20,7 @@ use anyhow::Result; use itertools::Itertools; use std::fs; use std::path::PathBuf; +use std::process::Command; use super::parser::ConfigType; @@ -93,11 +94,18 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { s.push_str(" @overload\n def __init__(self, scheme:str, **kwargs: str) -> None: ...\n"); - fs::write( - project_root.join("bindings/python/python/opendal/__base.pyi"), - s, - ) - .expect("failed to write result to file"); + let output_file: String = project_root + .join("bindings/python/python/opendal/__base.pyi") + .to_str() + .expect("should build output file path") + .into(); + + fs::write(output_file.clone(), s).expect("failed to write result to file"); + + Command::new("ruff") + .arg("format") + .arg(output_file) + .output()?; Ok(()) } From 5dae5feb60cf801800b232d31949ff5115dc299f Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 09:02:56 +0800 Subject: [PATCH 06/14] use template engine --- dev/Cargo.lock | 148 +++++++++++++++++++++ dev/Cargo.toml | 1 + dev/src/generate/binding_python.rs | 111 ++++++++-------- dev/templates/{python => python.py.jinja2} | 19 +++ 4 files changed, 225 insertions(+), 54 deletions(-) rename dev/templates/{python => python.py.jinja2} (74%) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index d1a8e49471ac..479563c2a974 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -66,6 +66,65 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "askama" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79091df18a97caea757e28cd2d5fda49c6cd4bd01ddffd7ff01ace0c0ad2c28" +dependencies = [ + "askama_derive", + "askama_escape", + "humansize", + "num-traits", + "percent-encoding", +] + +[[package]] +name = "askama_derive" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19fe8d6cb13c4714962c072ea496f3392015f0989b1a2847bb4b2d9effd71d83" +dependencies = [ + "askama_parser", + "basic-toml", + "mime", + "mime_guess", + "proc-macro2", + "quote", + "serde", + "syn", +] + +[[package]] +name = "askama_escape" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" + +[[package]] +name = "askama_parser" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acb1161c6b64d1c3d83108213c2a2533a342ac225aabd0bda218278c2ddb00c0" +dependencies = [ + "nom", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "basic-toml" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "823388e228f614e9558c6804262db37960ec8821856535f5c3f59913140558f8" +dependencies = [ + "serde", +] + [[package]] name = "clap" version = "4.5.23" @@ -162,6 +221,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.1.0" @@ -183,6 +251,12 @@ dependencies = [ "either", ] +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "log" version = "0.4.22" @@ -195,11 +269,53 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "odev" version = "0.0.1" dependencies = [ "anyhow", + "askama", "clap", "enquote", "env_logger", @@ -210,6 +326,12 @@ dependencies = [ "syn", ] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -267,6 +389,26 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "strsim" version = "0.11.1" @@ -304,6 +446,12 @@ dependencies = [ "syn", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 31120d77f674..54d2d664dc3c 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -29,6 +29,7 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" +askama = "0.12.1" clap = { version = "4.5.23", features = ["derive"] } enquote = "1.1.0" env_logger = "0.11.6" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index ad1ffa0d47ec..4660d4f572c3 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,12 +17,20 @@ use crate::generate::parser::Services; use anyhow::Result; +use askama::Template; use itertools::Itertools; -use std::fs; use std::path::PathBuf; use std::process::Command; +use std::{fs, iter}; // bring trait in scope -use super::parser::ConfigType; +use super::parser::{ConfigType, Service}; + +#[derive(Template)] // this will generate the code... +#[template(path = "python.py.jinja2")] // using the template in this path, relative + // to the `templates` dir in the crate root +struct PythonTemplate { + services: Vec<(String, Service)>, +} /// TODO: add a common utils to parse enabled features from cargo.toml fn enabled_service(srv: &str) -> bool { @@ -34,65 +42,34 @@ fn enabled_service(srv: &str) -> bool { } pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { - let mut s = fs::read_to_string(project_root.join("dev/templates/python")) - .expect("failed to read python template file"); - - for (srv, config) in services.clone().into_iter() { - if !enabled_service(srv.as_str()) { - continue; - } - - s.push_str("\n @overload\n"); - s.push_str(" def __init__(self,\n"); - s.push_str(format!("scheme: Literal[\"{}\"],", srv).as_str()); - s.push_str("\n*,\n"); + let mut v = Vec::from_iter( + services + .clone() + .into_iter() + .filter(|x| enabled_service(x.0.as_str())), + ); - for (_, f) in config + for mut srv in &mut v { + srv.1.config = srv + .1 .config + .clone() .into_iter() .enumerate() .sorted_by_key(|(i, x)| (x.optional, *i)) - { - if let Some(deprecated) = f.deprecated { - s.push_str("# deprecated: "); - s.push_str(deprecated.as_str()); - s.push('\n'); - } + .map(|(_, e)| e) + .collect(); + } - s.push_str(&f.name); - s.push_str(": "); - match f.value { - ConfigType::Bool => { - s.push_str("_bool"); - } - ConfigType::Duration => { - s.push_str("_duration"); - } - ConfigType::I64 - | ConfigType::Usize - | ConfigType::U64 - | ConfigType::U32 - | ConfigType::U16 => { - s.push_str("_int"); - } - ConfigType::Vec => { - s.push_str("_strings"); - } - ConfigType::String => { - s.push_str("str"); - } - } - if f.optional { - s.push_str(" = ...,\n"); - } else { - s.push_str(",\n"); - } - } + let tmpl = PythonTemplate { services: v }; - s.push_str(")->None:...\n"); - } + // for (srv, config) in services.clone().into_iter() { + // for f in config.config { +// f.deprecated.is_some() + // } + // } - s.push_str(" @overload\n def __init__(self, scheme:str, **kwargs: str) -> None: ...\n"); + let t = tmpl.render().expect("should render template"); let output_file: String = project_root .join("bindings/python/python/opendal/__base.pyi") @@ -100,7 +77,7 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { .expect("should build output file path") .into(); - fs::write(output_file.clone(), s).expect("failed to write result to file"); + fs::write(output_file.clone(), t).expect("failed to write result to file"); Command::new("ruff") .arg("format") @@ -109,3 +86,29 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { Ok(()) } + +impl ConfigType { + pub fn python_type(&self) -> String { + match self { + ConfigType::Bool => { + return "_bool".into(); + } + ConfigType::Duration => { + return "_duration".into(); + } + ConfigType::I64 + | ConfigType::Usize + | ConfigType::U64 + | ConfigType::U32 + | ConfigType::U16 => { + return "_int".into(); + } + ConfigType::Vec => { + return "_strings".into(); + } + ConfigType::String => { + return "str".into(); + } + } + } +} diff --git a/dev/templates/python b/dev/templates/python.py.jinja2 similarity index 74% rename from dev/templates/python rename to dev/templates/python.py.jinja2 index f0e662fa0ab9..34408fe6ec2b 100644 --- a/dev/templates/python +++ b/dev/templates/python.py.jinja2 @@ -42,3 +42,22 @@ class _Base: The services list here is support by opendal pypi wheel. """ + +{% for srv in services %} + @overload + def __init__(self, + scheme: Literal["{{srv.0}}"], + *, + + {% for field in srv.1.config.clone().into_iter() %} + {% if field.deprecated.is_some() %} + # deprecated: {{ field.deprecated.unwrap() }} + {% endif %} + {{field.name}}: {{field.value.python_type()}} {% if (field.optional) %} = ... {% endif %}, + {% endfor %} + + ) -> None: ... +{% endfor %} + + @overload + def __init__(self, scheme:str, **kwargs: str) -> None: ... From 7acb82b29fe25144c5082c009256bfb0e51e0abd Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 09:06:22 +0800 Subject: [PATCH 07/14] fix lint warning --- dev/src/generate/binding_python.rs | 23 +++++++++-------------- dev/src/generate/parser.rs | 14 ++++++-------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 4660d4f572c3..a95609b8e772 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -19,9 +19,9 @@ use crate::generate::parser::Services; use anyhow::Result; use askama::Template; use itertools::Itertools; +use std::fs; use std::path::PathBuf; use std::process::Command; -use std::{fs, iter}; // bring trait in scope use super::parser::{ConfigType, Service}; @@ -49,7 +49,7 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { .filter(|x| enabled_service(x.0.as_str())), ); - for mut srv in &mut v { + for srv in &mut v { srv.1.config = srv .1 .config @@ -63,12 +63,6 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { let tmpl = PythonTemplate { services: v }; - // for (srv, config) in services.clone().into_iter() { - // for f in config.config { -// f.deprecated.is_some() - // } - // } - let t = tmpl.render().expect("should render template"); let output_file: String = project_root @@ -82,7 +76,8 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { Command::new("ruff") .arg("format") .arg(output_file) - .output()?; + .output() + .expect("failed to run ruff"); Ok(()) } @@ -91,23 +86,23 @@ impl ConfigType { pub fn python_type(&self) -> String { match self { ConfigType::Bool => { - return "_bool".into(); + "_bool".into() } ConfigType::Duration => { - return "_duration".into(); + "_duration".into() } ConfigType::I64 | ConfigType::Usize | ConfigType::U64 | ConfigType::U32 | ConfigType::U16 => { - return "_int".into(); + "_int".into() } ConfigType::Vec => { - return "_strings".into(); + "_strings".into() } ConfigType::String => { - return "str".into(); + "str".into() } } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 6a265f2f2c14..2f721d88e196 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -183,7 +183,7 @@ impl ServiceParser { let mut config = Vec::with_capacity(config_struct.fields.len()); for field in config_struct.fields { - let field = self.parse_field(field)?; + let field = Self::parse_field(field)?; config.push(field); } @@ -192,13 +192,13 @@ impl ServiceParser { } /// TODO: Add comment parse support. - fn parse_field(&self, field: Field) -> Result { + fn parse_field(field: Field) -> Result { let name = field .ident .clone() .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; - let deprecated = self.deprecated_note(&field); + let deprecated = Self::deprecated_note(&field); let (cfg_type, optional) = match &field.ty { Type::Path(TypePath { path, .. }) => { @@ -246,7 +246,7 @@ impl ServiceParser { }) } - fn deprecated_note(&self, field: &Field) -> Option { + fn deprecated_note(field: &Field) -> Option { for attr in &field.attrs { if !attr.path().is_ident("deprecated") { continue; @@ -261,9 +261,7 @@ impl ServiceParser { for (index, token) in tokens.iter().enumerate() { let ident = match token { proc_macro2::TokenTree::Ident(ident) => ident, - _ => { - continue; - } + _ => continue, }; if ident == "note" { @@ -277,7 +275,7 @@ impl ServiceParser { } } - return None; + None } } From 96dd19cf849604ce436511176a702e729cb26bbb Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 09:08:18 +0800 Subject: [PATCH 08/14] comment --- dev/src/generate/binding_python.rs | 21 ++++++--------------- dev/templates/python.py.jinja2 | 2 +- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index a95609b8e772..864f5ae43672 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -49,6 +49,7 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { .filter(|x| enabled_service(x.0.as_str())), ); + // move required options at beginning. for srv in &mut v { srv.1.config = srv .1 @@ -85,25 +86,15 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { impl ConfigType { pub fn python_type(&self) -> String { match self { - ConfigType::Bool => { - "_bool".into() - } - ConfigType::Duration => { - "_duration".into() - } + ConfigType::Bool => "_bool".into(), + ConfigType::Duration => "_duration".into(), ConfigType::I64 | ConfigType::Usize | ConfigType::U64 | ConfigType::U32 - | ConfigType::U16 => { - "_int".into() - } - ConfigType::Vec => { - "_strings".into() - } - ConfigType::String => { - "str".into() - } + | ConfigType::U16 => "_int".into(), + ConfigType::Vec => "_strings".into(), + ConfigType::String => "str".into(), } } } diff --git a/dev/templates/python.py.jinja2 b/dev/templates/python.py.jinja2 index 34408fe6ec2b..f5525de60a87 100644 --- a/dev/templates/python.py.jinja2 +++ b/dev/templates/python.py.jinja2 @@ -53,7 +53,7 @@ class _Base: {% if field.deprecated.is_some() %} # deprecated: {{ field.deprecated.unwrap() }} {% endif %} - {{field.name}}: {{field.value.python_type()}} {% if (field.optional) %} = ... {% endif %}, + {{field.name}}: {{field.value.python_type()}} {% if field.optional %} = ... {% endif %}, {% endfor %} ) -> None: ... From afbc78f740557ef79215414d9cea6939ef367099 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 09:38:49 +0800 Subject: [PATCH 09/14] avoid collision --- bindings/python/python/opendal/__base.pyi | 63 ++++++++++++++++++++++- dev/templates/python.py.jinja2 | 5 +- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/bindings/python/python/opendal/__base.pyi b/bindings/python/python/opendal/__base.pyi index d18fbed11cdb..c9e794b81985 100644 --- a/bindings/python/python/opendal/__base.pyi +++ b/bindings/python/python/opendal/__base.pyi @@ -46,6 +46,7 @@ class _Base: def __init__( self, scheme: Literal["aliyun_drive"], + /, *, drive_type: str, root: str = ..., @@ -58,6 +59,7 @@ class _Base: def __init__( self, scheme: Literal["alluxio"], + /, *, root: str = ..., endpoint: str = ..., @@ -66,6 +68,7 @@ class _Base: def __init__( self, scheme: Literal["atomicserver"], + /, *, root: str = ..., endpoint: str = ..., @@ -77,6 +80,7 @@ class _Base: def __init__( self, scheme: Literal["azblob"], + /, *, container: str, root: str = ..., @@ -93,6 +97,7 @@ class _Base: def __init__( self, scheme: Literal["azdls"], + /, *, filesystem: str, root: str = ..., @@ -104,6 +109,7 @@ class _Base: def __init__( self, scheme: Literal["azfile"], + /, *, share_name: str, root: str = ..., @@ -116,6 +122,7 @@ class _Base: def __init__( self, scheme: Literal["b2"], + /, *, bucket: str, bucket_id: str, @@ -127,6 +134,7 @@ class _Base: def __init__( self, scheme: Literal["cacache"], + /, *, datadir: str = ..., ) -> None: ... @@ -134,6 +142,7 @@ class _Base: def __init__( self, scheme: Literal["chainsafe"], + /, *, bucket_id: str, root: str = ..., @@ -143,6 +152,7 @@ class _Base: def __init__( self, scheme: Literal["cloudflare_kv"], + /, *, token: str = ..., account_id: str = ..., @@ -153,6 +163,7 @@ class _Base: def __init__( self, scheme: Literal["compfs"], + /, *, root: str = ..., ) -> None: ... @@ -160,6 +171,7 @@ class _Base: def __init__( self, scheme: Literal["cos"], + /, *, root: str = ..., endpoint: str = ..., @@ -172,6 +184,7 @@ class _Base: def __init__( self, scheme: Literal["d1"], + /, *, token: str = ..., account_id: str = ..., @@ -185,6 +198,7 @@ class _Base: def __init__( self, scheme: Literal["dashmap"], + /, *, root: str = ..., ) -> None: ... @@ -192,6 +206,7 @@ class _Base: def __init__( self, scheme: Literal["dbfs"], + /, *, root: str = ..., endpoint: str = ..., @@ -201,6 +216,7 @@ class _Base: def __init__( self, scheme: Literal["dropbox"], + /, *, root: str = ..., access_token: str = ..., @@ -212,6 +228,7 @@ class _Base: def __init__( self, scheme: Literal["fs"], + /, *, root: str = ..., atomic_write_dir: str = ..., @@ -220,6 +237,7 @@ class _Base: def __init__( self, scheme: Literal["gcs"], + /, *, bucket: str, root: str = ..., @@ -239,6 +257,7 @@ class _Base: def __init__( self, scheme: Literal["gdrive"], + /, *, root: str = ..., access_token: str = ..., @@ -250,6 +269,7 @@ class _Base: def __init__( self, scheme: Literal["ghac"], + /, *, root: str = ..., version: str = ..., @@ -260,6 +280,7 @@ class _Base: def __init__( self, scheme: Literal["github"], + /, *, owner: str, repo: str, @@ -270,6 +291,7 @@ class _Base: def __init__( self, scheme: Literal["gridfs"], + /, *, connection_string: str = ..., database: str = ..., @@ -281,6 +303,7 @@ class _Base: def __init__( self, scheme: Literal["hdfs_native"], + /, *, root: str = ..., url: str = ..., @@ -290,6 +313,7 @@ class _Base: def __init__( self, scheme: Literal["http"], + /, *, endpoint: str = ..., username: str = ..., @@ -301,6 +325,7 @@ class _Base: def __init__( self, scheme: Literal["huggingface"], + /, *, repo_type: str = ..., repo_id: str = ..., @@ -312,6 +337,7 @@ class _Base: def __init__( self, scheme: Literal["icloud"], + /, *, root: str = ..., apple_id: str = ..., @@ -324,6 +350,7 @@ class _Base: def __init__( self, scheme: Literal["ipfs"], + /, *, endpoint: str = ..., root: str = ..., @@ -332,6 +359,7 @@ class _Base: def __init__( self, scheme: Literal["ipmfs"], + /, *, root: str = ..., endpoint: str = ..., @@ -340,6 +368,7 @@ class _Base: def __init__( self, scheme: Literal["koofr"], + /, *, endpoint: str, email: str, @@ -350,6 +379,7 @@ class _Base: def __init__( self, scheme: Literal["lakefs"], + /, *, endpoint: str = ..., username: str = ..., @@ -362,6 +392,7 @@ class _Base: def __init__( self, scheme: Literal["libsql"], + /, *, connection_string: str = ..., auth_token: str = ..., @@ -374,6 +405,7 @@ class _Base: def __init__( self, scheme: Literal["memcached"], + /, *, endpoint: str = ..., root: str = ..., @@ -385,6 +417,7 @@ class _Base: def __init__( self, scheme: Literal["memory"], + /, *, root: str = ..., ) -> None: ... @@ -392,6 +425,7 @@ class _Base: def __init__( self, scheme: Literal["mini_moka"], + /, *, max_capacity: _int = ..., time_to_live: _duration = ..., @@ -402,6 +436,7 @@ class _Base: def __init__( self, scheme: Literal["moka"], + /, *, name: str = ..., max_capacity: _int = ..., @@ -414,6 +449,7 @@ class _Base: def __init__( self, scheme: Literal["mongodb"], + /, *, connection_string: str = ..., database: str = ..., @@ -426,6 +462,7 @@ class _Base: def __init__( self, scheme: Literal["monoiofs"], + /, *, root: str = ..., ) -> None: ... @@ -433,6 +470,7 @@ class _Base: def __init__( self, scheme: Literal["mysql"], + /, *, connection_string: str = ..., table: str = ..., @@ -444,6 +482,7 @@ class _Base: def __init__( self, scheme: Literal["nebula_graph"], + /, *, host: str = ..., port: _int = ..., @@ -459,6 +498,7 @@ class _Base: def __init__( self, scheme: Literal["obs"], + /, *, root: str = ..., endpoint: str = ..., @@ -470,6 +510,7 @@ class _Base: def __init__( self, scheme: Literal["onedrive"], + /, *, access_token: str = ..., root: str = ..., @@ -478,6 +519,7 @@ class _Base: def __init__( self, scheme: Literal["oss"], + /, *, bucket: str, root: str = ..., @@ -501,6 +543,7 @@ class _Base: def __init__( self, scheme: Literal["pcloud"], + /, *, endpoint: str, root: str = ..., @@ -511,6 +554,7 @@ class _Base: def __init__( self, scheme: Literal["persy"], + /, *, datafile: str = ..., segment: str = ..., @@ -520,6 +564,7 @@ class _Base: def __init__( self, scheme: Literal["postgresql"], + /, *, root: str = ..., connection_string: str = ..., @@ -531,6 +576,7 @@ class _Base: def __init__( self, scheme: Literal["redb"], + /, *, datadir: str = ..., root: str = ..., @@ -540,6 +586,7 @@ class _Base: def __init__( self, scheme: Literal["redis"], + /, *, db: _int, endpoint: str = ..., @@ -553,6 +600,7 @@ class _Base: def __init__( self, scheme: Literal["s3"], + /, *, bucket: str, root: str = ..., @@ -586,6 +634,7 @@ class _Base: def __init__( self, scheme: Literal["seafile"], + /, *, repo_name: str, root: str = ..., @@ -597,6 +646,7 @@ class _Base: def __init__( self, scheme: Literal["sftp"], + /, *, endpoint: str = ..., root: str = ..., @@ -609,6 +659,7 @@ class _Base: def __init__( self, scheme: Literal["sled"], + /, *, datadir: str = ..., root: str = ..., @@ -618,6 +669,7 @@ class _Base: def __init__( self, scheme: Literal["sqlite"], + /, *, connection_string: str = ..., table: str = ..., @@ -629,6 +681,7 @@ class _Base: def __init__( self, scheme: Literal["supabase"], + /, *, bucket: str, root: str = ..., @@ -639,6 +692,7 @@ class _Base: def __init__( self, scheme: Literal["surrealdb"], + /, *, connection_string: str = ..., username: str = ..., @@ -654,6 +708,7 @@ class _Base: def __init__( self, scheme: Literal["swift"], + /, *, endpoint: str = ..., container: str = ..., @@ -664,6 +719,7 @@ class _Base: def __init__( self, scheme: Literal["upyun"], + /, *, bucket: str, root: str = ..., @@ -674,6 +730,7 @@ class _Base: def __init__( self, scheme: Literal["vercel_artifacts"], + /, *, access_token: str = ..., ) -> None: ... @@ -681,6 +738,7 @@ class _Base: def __init__( self, scheme: Literal["vercel_blob"], + /, *, root: str = ..., token: str = ..., @@ -689,6 +747,7 @@ class _Base: def __init__( self, scheme: Literal["webdav"], + /, *, endpoint: str = ..., username: str = ..., @@ -701,6 +760,7 @@ class _Base: def __init__( self, scheme: Literal["webhdfs"], + /, *, root: str = ..., endpoint: str = ..., @@ -712,9 +772,10 @@ class _Base: def __init__( self, scheme: Literal["yandex_disk"], + /, *, access_token: str, root: str = ..., ) -> None: ... @overload - def __init__(self, scheme: str, **kwargs: str) -> None: ... + def __init__(self, scheme: str, /, **kwargs: str) -> None: ... diff --git a/dev/templates/python.py.jinja2 b/dev/templates/python.py.jinja2 index f5525de60a87..40cf8f1f03f4 100644 --- a/dev/templates/python.py.jinja2 +++ b/dev/templates/python.py.jinja2 @@ -47,17 +47,16 @@ class _Base: @overload def __init__(self, scheme: Literal["{{srv.0}}"], + /, *, - {% for field in srv.1.config.clone().into_iter() %} {% if field.deprecated.is_some() %} # deprecated: {{ field.deprecated.unwrap() }} {% endif %} {{field.name}}: {{field.value.python_type()}} {% if field.optional %} = ... {% endif %}, {% endfor %} - ) -> None: ... {% endfor %} @overload - def __init__(self, scheme:str, **kwargs: str) -> None: ... + def __init__(self, scheme:str, /, **kwargs: str) -> None: ... From e2db0cb8148686f3e52182489627170643c3ce21 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 09:45:50 +0800 Subject: [PATCH 10/14] disable default escape --- dev/src/generate/binding_python.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 864f5ae43672..f5a3fd26ac6d 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -25,9 +25,10 @@ use std::process::Command; use super::parser::{ConfigType, Service}; -#[derive(Template)] // this will generate the code... -#[template(path = "python.py.jinja2")] // using the template in this path, relative - // to the `templates` dir in the crate root +// Using the template in this path, relative +// to the `templates` dir in the crate root +#[derive(Template)] +#[template(path = "python.py.jinja2", escape = "none")] struct PythonTemplate { services: Vec<(String, Service)>, } From cf174690e9c5ee171805f41672ee3abd7db4fa03 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 10:01:02 +0800 Subject: [PATCH 11/14] use rinja --- dev/Cargo.lock | 114 ++++++++++++++--------------- dev/Cargo.toml | 2 +- dev/src/generate/binding_python.rs | 2 +- 3 files changed, 56 insertions(+), 62 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index 479563c2a974..a176b69182e0 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -66,56 +66,6 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" -[[package]] -name = "askama" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79091df18a97caea757e28cd2d5fda49c6cd4bd01ddffd7ff01ace0c0ad2c28" -dependencies = [ - "askama_derive", - "askama_escape", - "humansize", - "num-traits", - "percent-encoding", -] - -[[package]] -name = "askama_derive" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19fe8d6cb13c4714962c072ea496f3392015f0989b1a2847bb4b2d9effd71d83" -dependencies = [ - "askama_parser", - "basic-toml", - "mime", - "mime_guess", - "proc-macro2", - "quote", - "serde", - "syn", -] - -[[package]] -name = "askama_escape" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" - -[[package]] -name = "askama_parser" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acb1161c6b64d1c3d83108213c2a2533a342ac225aabd0bda218278c2ddb00c0" -dependencies = [ - "nom", -] - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - [[package]] name = "basic-toml" version = "0.1.9" @@ -251,6 +201,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + [[package]] name = "libm" version = "0.2.11" @@ -301,21 +257,11 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "odev" version = "0.0.1" dependencies = [ "anyhow", - "askama", "clap", "enquote", "env_logger", @@ -323,6 +269,7 @@ dependencies = [ "log", "pretty_assertions", "proc-macro2", + "rinja", "syn", ] @@ -389,6 +336,53 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rinja" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5" +dependencies = [ + "humansize", + "itoa", + "percent-encoding", + "rinja_derive", +] + +[[package]] +name = "rinja_derive" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b" +dependencies = [ + "basic-toml", + "memchr", + "mime", + "mime_guess", + "proc-macro2", + "quote", + "rinja_parser", + "rustc-hash", + "serde", + "syn", +] + +[[package]] +name = "rinja_parser" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610" +dependencies = [ + "memchr", + "nom", + "serde", +] + +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "serde" version = "1.0.216" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 54d2d664dc3c..e7b912a0bda0 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -29,13 +29,13 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" -askama = "0.12.1" clap = { version = "4.5.23", features = ["derive"] } enquote = "1.1.0" env_logger = "0.11.6" itertools = "0.13.0" log = "0.4.22" proc-macro2 = { version = "1.0.91", features = ["span-locations"] } +rinja = "0.3.5" syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index f5a3fd26ac6d..b41f361d074b 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,7 +17,7 @@ use crate::generate::parser::Services; use anyhow::Result; -use askama::Template; +use rinja::Template; use itertools::Itertools; use std::fs; use std::path::PathBuf; From 1a43a06310fe9b5caa90405f7bdcd39a778649d6 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 13:43:32 +0800 Subject: [PATCH 12/14] make ruff format optional --- dev/src/generate/binding_python.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 3e024855418d..33deb5e62fa9 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -70,11 +70,11 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { fs::write(output_file.clone(), t).expect("failed to write result to file"); - Command::new("ruff") - .arg("format") - .arg(output_file) - .output() - .expect("failed to run ruff"); + let ruff = Command::new("ruff").arg("format").arg(output_file).output(); + + if let Err(err) = ruff { + println!("failed to format generate code with ruff: {}", err); + } Ok(()) } From c5861612ffdfbe23a45724bb2522eb9103475644 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 13:44:28 +0800 Subject: [PATCH 13/14] disable ruff cache --- dev/src/generate/binding_python.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 33deb5e62fa9..aea5bd83de0e 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -70,7 +70,11 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { fs::write(output_file.clone(), t).expect("failed to write result to file"); - let ruff = Command::new("ruff").arg("format").arg(output_file).output(); + let ruff = Command::new("ruff") + .arg("format") + .arg("--no-cache") + .arg(output_file) + .output(); if let Err(err) = ruff { println!("failed to format generate code with ruff: {}", err); From 794288eed0e7b360ecd5f8faed035bc1e4cdda22 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 13:47:35 +0800 Subject: [PATCH 14/14] remove ruff --- dev/src/generate/binding_python.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index aea5bd83de0e..68bd9b4577d5 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -20,7 +20,6 @@ use anyhow::Result; use rinja::Template; use std::fs; use std::path::PathBuf; -use std::process::Command; use super::parser::{ConfigType, Service}; @@ -68,17 +67,7 @@ pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { .expect("should build output file path") .into(); - fs::write(output_file.clone(), t).expect("failed to write result to file"); - - let ruff = Command::new("ruff") - .arg("format") - .arg("--no-cache") - .arg(output_file) - .output(); - - if let Err(err) = ruff { - println!("failed to format generate code with ruff: {}", err); - } + fs::write(output_file, t).expect("failed to write result to file"); Ok(()) }