diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index a2db6361c85d..5d5067b8bbaf 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -45,6 +45,9 @@ default = [ "services-webhdfs", ] +# NOTE: this is the feature we used to build pypi wheels. +# When enable it or disable some feature, +# also need to update dev/src/generate/binding_python.rs `enabled_service` to match it. services-all = [ "default", "services-aliyun-drive", @@ -160,9 +163,7 @@ doc = false bytes = "1.5.0" futures = "0.3.28" # this crate won't be published, we always use the local version -opendal = { version = ">=0", path = "../../core", features = [ - "layers-blocking", -] } +opendal = { version = ">=0", path = "../../core", features = ["layers-blocking"] } pyo3 = { version = "0.23.3", features = ["generate-import-lib"] } pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] } tokio = "1" diff --git a/bindings/python/python/opendal/__base.pyi b/bindings/python/python/opendal/__base.pyi new file mode 100644 index 000000000000..d18fbed11cdb --- /dev/null +++ b/bindings/python/python/opendal/__base.pyi @@ -0,0 +1,720 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """ + + @overload + def __init__( + self, + scheme: Literal["aliyun_drive"], + *, + drive_type: str, + root: str = ..., + access_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + refresh_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["alluxio"], + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["atomicserver"], + *, + root: str = ..., + endpoint: str = ..., + private_key: str = ..., + public_key: str = ..., + parent_resource_id: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azblob"], + *, + container: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + encryption_key: str = ..., + encryption_key_sha256: str = ..., + encryption_algorithm: str = ..., + sas_token: str = ..., + batch_max_operations: _int = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azdls"], + *, + filesystem: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azfile"], + *, + share_name: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + sas_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["b2"], + *, + bucket: str, + bucket_id: str, + root: str = ..., + application_key_id: str = ..., + application_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cacache"], + *, + datadir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["chainsafe"], + *, + bucket_id: str, + root: str = ..., + api_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cloudflare_kv"], + *, + token: str = ..., + account_id: str = ..., + namespace_id: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["compfs"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cos"], + *, + root: str = ..., + endpoint: str = ..., + secret_id: str = ..., + secret_key: str = ..., + bucket: str = ..., + disable_config_load: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["d1"], + *, + token: str = ..., + account_id: str = ..., + database_id: str = ..., + root: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dashmap"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dbfs"], + *, + root: str = ..., + endpoint: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dropbox"], + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["fs"], + *, + root: str = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gcs"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + scope: str = ..., + service_account: str = ..., + credential: str = ..., + credential_path: str = ..., + predefined_acl: str = ..., + default_storage_class: str = ..., + allow_anonymous: _bool = ..., + disable_vm_metadata: _bool = ..., + disable_config_load: _bool = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gdrive"], + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ghac"], + *, + root: str = ..., + version: str = ..., + endpoint: str = ..., + runtime_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["github"], + *, + owner: str, + repo: str, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gridfs"], + *, + connection_string: str = ..., + database: str = ..., + bucket: str = ..., + chunk_size: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["hdfs_native"], + *, + root: str = ..., + url: str = ..., + enable_append: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["http"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["huggingface"], + *, + repo_type: str = ..., + repo_id: str = ..., + revision: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["icloud"], + *, + root: str = ..., + apple_id: str = ..., + password: str = ..., + trust_token: str = ..., + ds_web_auth_token: str = ..., + is_china_mainland: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipfs"], + *, + endpoint: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipmfs"], + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["koofr"], + *, + endpoint: str, + email: str, + root: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["lakefs"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + repository: str = ..., + branch: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["libsql"], + *, + connection_string: str = ..., + auth_token: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memcached"], + *, + endpoint: str = ..., + root: str = ..., + username: str = ..., + password: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memory"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mini_moka"], + *, + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["moka"], + *, + name: str = ..., + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + num_segments: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mongodb"], + *, + connection_string: str = ..., + database: str = ..., + collection: str = ..., + root: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["monoiofs"], + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mysql"], + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["nebula_graph"], + *, + host: str = ..., + port: _int = ..., + username: str = ..., + password: str = ..., + space: str = ..., + tag: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["obs"], + *, + root: str = ..., + endpoint: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + bucket: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["onedrive"], + *, + access_token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["oss"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + presign_endpoint: str = ..., + server_side_encryption: str = ..., + server_side_encryption_key_id: str = ..., + allow_anonymous: _bool = ..., + access_key_id: str = ..., + access_key_secret: str = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + role_arn: str = ..., + role_session_name: str = ..., + oidc_provider_arn: str = ..., + oidc_token_file: str = ..., + sts_endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["pcloud"], + *, + endpoint: str, + root: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["persy"], + *, + datafile: str = ..., + segment: str = ..., + index: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["postgresql"], + *, + root: str = ..., + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redb"], + *, + datadir: str = ..., + root: str = ..., + table: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redis"], + *, + db: _int, + endpoint: str = ..., + cluster_endpoints: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["s3"], + *, + bucket: str, + root: str = ..., + enable_versioning: _bool = ..., + endpoint: str = ..., + region: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + session_token: str = ..., + role_arn: str = ..., + external_id: str = ..., + role_session_name: str = ..., + disable_config_load: _bool = ..., + disable_ec2_metadata: _bool = ..., + allow_anonymous: _bool = ..., + server_side_encryption: str = ..., + server_side_encryption_aws_kms_key_id: str = ..., + server_side_encryption_customer_algorithm: str = ..., + server_side_encryption_customer_key: str = ..., + server_side_encryption_customer_key_md5: str = ..., + default_storage_class: str = ..., + enable_virtual_host_style: _bool = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + disable_stat_with_override: _bool = ..., + checksum_algorithm: str = ..., + disable_write_with_if_match: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["seafile"], + *, + repo_name: str, + root: str = ..., + endpoint: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sftp"], + *, + endpoint: str = ..., + root: str = ..., + user: str = ..., + key: str = ..., + known_hosts_strategy: str = ..., + enable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sled"], + *, + datadir: str = ..., + root: str = ..., + tree: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sqlite"], + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["supabase"], + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["surrealdb"], + *, + connection_string: str = ..., + username: str = ..., + password: str = ..., + namespace: str = ..., + database: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["swift"], + *, + endpoint: str = ..., + container: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["upyun"], + *, + bucket: str, + root: str = ..., + operator: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_artifacts"], + *, + access_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_blob"], + *, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webdav"], + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + disable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webhdfs"], + *, + root: str = ..., + endpoint: str = ..., + delegation: str = ..., + disable_list_batch: _bool = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["yandex_disk"], + *, + access_token: str, + root: str = ..., + ) -> None: ... + @overload + def __init__(self, scheme: str, **kwargs: str) -> None: ... diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi index 6d6645a60bbc..663cbc677c20 100644 --- a/bindings/python/python/opendal/__init__.pyi +++ b/bindings/python/python/opendal/__init__.pyi @@ -15,16 +15,16 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, AsyncIterable, Iterable, Optional, final, Union, Type +from typing import AsyncIterable, Iterable, Optional, final, Union, Type from types import TracebackType from opendal import exceptions as exceptions from opendal import layers as layers from opendal.layers import Layer +from opendal.__base import _Base @final -class Operator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class Operator(_Base): def layer(self, layer: Layer) -> "Operator": ... def open(self, path: str, mode: str) -> File: ... def read(self, path: str) -> bytes: ... @@ -51,8 +51,7 @@ class Operator: def to_async_operator(self) -> AsyncOperator: ... @final -class AsyncOperator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class AsyncOperator(_Base): def layer(self, layer: Layer) -> "AsyncOperator": ... async def open(self, path: str, mode: str) -> AsyncFile: ... async def read(self, path: str) -> bytes: ... diff --git a/dev/Cargo.lock b/dev/Cargo.lock index fff2ed814491..d1a8e49471ac 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -118,6 +118,21 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -159,6 +174,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "log" version = "0.4.22" @@ -177,7 +201,9 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", + "enquote", "env_logger", + "itertools", "log", "pretty_assertions", "proc-macro2", @@ -258,6 +284,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index ef5fbaa215e7..31120d77f674 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -30,10 +30,12 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } +enquote = "1.1.0" env_logger = "0.11.6" +itertools = "0.13.0" log = "0.4.22" proc-macro2 = { version = "1.0.91", features = ["span-locations"] } -syn = { version = "2.0.91", features = ["visit", "full", "extra-traits",'printing', 'parsing'] } +syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 7f3e3bdef83f..b03471715c60 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,9 +17,86 @@ use crate::generate::parser::Services; use anyhow::Result; +use itertools::Itertools; +use std::fs; +use std::path::PathBuf; -pub fn generate(services: &Services) -> Result<()> { - println!("{:?}", services); +use super::parser::ConfigType; + +fn enabled_service(srv: &str) -> bool { + match srv { + // not enabled in bindings/python/Cargo.toml + "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, + _ => true, + } +} + +pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { + let mut s = fs::read_to_string(project_root.join("dev/templates/python")) + .expect("failed to read python template file"); + + for (srv, config) in services.clone().into_iter() { + if !enabled_service(srv.as_str()) { + continue; + } + + s.push_str("\n @overload\n"); + s.push_str(" def __init__(self,\n"); + s.push_str(format!("scheme: Literal[\"{}\"],", srv).as_str()); + s.push_str("\n*,\n"); + + for (_, f) in config + .config + .into_iter() + .enumerate() + .sorted_by_key(|(i, x)| (x.optional, *i)) + { + if let Some(deprecated) = f.deprecated { + s.push_str("# deprecated: "); + s.push_str(deprecated.as_str()); + s.push('\n'); + } + + s.push_str(&f.name); + s.push_str(": "); + match f.value { + ConfigType::Bool => { + s.push_str("_bool"); + } + ConfigType::Duration => { + s.push_str("_duration"); + } + ConfigType::I64 + | ConfigType::Usize + | ConfigType::U64 + | ConfigType::U32 + | ConfigType::U16 => { + s.push_str("_int"); + } + ConfigType::Vec => { + s.push_str("_strings"); + } + ConfigType::String => { + s.push_str("str"); + } + } + if f.optional { + s.push_str(" = ...,\n"); + } else { + s.push_str(",\n"); + } + } + + s.push_str(")->None:...\n"); + } + + s.push_str(" @overload\n def __init__(self, scheme:str, **kwargs: str) -> None: ...\n"); + + fs::write( + project_root.join("bindings/python/python/opendal/__base.pyi"), + s, + ) + .expect("failed to write result to file"); Ok(()) } diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 743aa3b1c483..71ec44429998 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -25,10 +25,11 @@ use std::path::PathBuf; pub fn run(language: &str) -> Result<()> { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let services_path = manifest_dir.join("../core/src/services").canonicalize()?; + let project_root = manifest_dir.join("..").canonicalize()?; let services = parser::parse(&services_path.to_string_lossy())?; match language { - "python" | "py" => binding_python::generate(&services), + "python" | "py" => binding_python::generate(project_root, &services), _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 2d731bfcf550..83723e77de43 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -17,12 +17,12 @@ use anyhow::{anyhow, Context}; use anyhow::{Ok, Result}; +use itertools::Itertools; use log::debug; -use std::collections::hash_map; use std::collections::HashMap; -use std::fs; use std::fs::read_dir; use std::str::FromStr; +use std::{fs, vec}; use syn::{Field, GenericArgument, Item, PathArguments, Type, TypePath}; #[derive(Debug, Clone)] @@ -30,10 +30,10 @@ pub struct Services(HashMap); impl IntoIterator for Services { type Item = (String, Service); - type IntoIter = hash_map::IntoIter; + type IntoIter = vec::IntoIter<(String, Service)>; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + self.0.into_iter().sorted_by_key(|x| x.0.clone()) } } @@ -213,7 +213,12 @@ impl ServiceParser { proc_macro2::TokenTree::Ident(i) => { if i == "note" { deprecated = - tokens.get(index + 2).unwrap().span().source_text(); + tokens.get(index + 2).unwrap().span().source_text().map( + |s| { + enquote::unquote(s.as_str()) + .expect("should unquote string") + }, + ); break; } } @@ -257,14 +262,13 @@ impl ServiceParser { }; let typ = type_name.as_str().parse()?; + let optional = optional || typ == ConfigType::Bool; (typ, optional) } v => return Err(anyhow!("unsupported config type {v:?}")), }; - deprecated = deprecated.map(|s| s[1..s.len() - 1].into()); - Ok(Config { name: name.to_string(), value: cfg_type, @@ -550,7 +554,7 @@ impl Debug for S3Config { Config { name: "disable_write_with_if_match".to_string(), value: ConfigType::Bool, - optional: false, + optional: true, deprecated: None, comments: "".to_string(), }, diff --git a/dev/templates/python b/dev/templates/python new file mode 100644 index 000000000000..f0e662fa0ab9 --- /dev/null +++ b/dev/templates/python @@ -0,0 +1,44 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """