diff --git a/codegen/.gitignore b/codegen/.gitignore new file mode 100644 index 000000000000..3a8816c9ee24 --- /dev/null +++ b/codegen/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm-project.org/#use-with-ide +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/codegen/README.md b/codegen/README.md new file mode 100644 index 000000000000..ece9165bf4f2 --- /dev/null +++ b/codegen/README.md @@ -0,0 +1,21 @@ +# Codegen + +## Generate + +### Generate service config files + +Run the following command to generate service config files: + +```bash +pdm genconfig +``` + +## Develop + +### Check code style and typing + +Run the following command to invoke linter, formatter, and type checker. + +```bash +pdm check +``` diff --git a/codegen/bin/genconfig.py b/codegen/bin/genconfig.py new file mode 100644 index 000000000000..2555d0f8cf6b --- /dev/null +++ b/codegen/bin/genconfig.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader + +from codegen.config import S3 + +if __name__ == '__main__': + codegen = Path(__file__).resolve().parent.parent + env = Environment( + loader=FileSystemLoader(codegen / 'template'), + ) + tmpl = env.get_template("config.rs.j2") + + opendal = codegen.parent + rust = opendal / 'core' / 'src' / 'services' / 'config.rs' + with rust.open('w') as f: + assert tmpl.filename is not None + template = Path(tmpl.filename).relative_to(opendal) + output = rust.relative_to(opendal) + print(f'Generating Rust files from templates in {template} to {output}') + f.write(tmpl.render(configs=[S3])) + + subprocess.run(['rustfmt', rust]) diff --git a/codegen/pdm.lock b/codegen/pdm.lock new file mode 100644 index 000000000000..2f3e8301138d --- /dev/null +++ b/codegen/pdm.lock @@ -0,0 +1,117 @@ +# This file is @generated by PDM. +# It is not intended for manual editing. + +[metadata] +groups = ["default", "check", "bin"] +strategy = ["cross_platform", "inherit_metadata"] +lock_version = "4.4.1" +content_hash = "sha256:c4d36f2e6cd71fb5450daadd6f9f1912a2693229a3b40b1948c22a299bd49344" + +[[package]] +name = "jinja2" +version = "3.1.3" +requires_python = ">=3.7" +summary = "A very fast and expressive template engine." +groups = ["bin"] +dependencies = [ + "MarkupSafe>=2.0", +] +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[[package]] +name = "markupsafe" +version = "2.1.5" +requires_python = ">=3.7" +summary = "Safely add untrusted strings to HTML/XML markup." +groups = ["bin"] +files = [ + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "nodeenv" +version = "1.8.0" +requires_python = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +summary = "Node.js virtual environment builder" +groups = ["check"] +dependencies = [ + "setuptools", +] +files = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] + +[[package]] +name = "pyhumps" +version = "3.8.0" +summary = "🐫 Convert strings (and dictionary keys) between snake case, camel case and pascal case in Python. Inspired by Humps for Node" +groups = ["default"] +files = [ + {file = "pyhumps-3.8.0-py3-none-any.whl", hash = "sha256:060e1954d9069f428232a1adda165db0b9d8dfdce1d265d36df7fbff540acfd6"}, + {file = "pyhumps-3.8.0.tar.gz", hash = "sha256:498026258f7ee1a8e447c2e28526c0bea9407f9a59c03260aee4bd6c04d681a3"}, +] + +[[package]] +name = "pyright" +version = "1.1.354" +requires_python = ">=3.7" +summary = "Command line wrapper for pyright" +groups = ["check"] +dependencies = [ + "nodeenv>=1.6.0", +] +files = [ + {file = "pyright-1.1.354-py3-none-any.whl", hash = "sha256:f28d61ae8ae035fc52ded1070e8d9e786051a26a4127bbd7a4ba0399b81b37b5"}, + {file = "pyright-1.1.354.tar.gz", hash = "sha256:b1070dc774ff2e79eb0523fe87f4ba9a90550de7e4b030a2bc9e031864029a1f"}, +] + +[[package]] +name = "ruff" +version = "0.3.3" +requires_python = ">=3.7" +summary = "An extremely fast Python linter and code formatter, written in Rust." +groups = ["check"] +files = [ + {file = "ruff-0.3.3-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:973a0e388b7bc2e9148c7f9be8b8c6ae7471b9be37e1cc732f8f44a6f6d7720d"}, + {file = "ruff-0.3.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:cfa60d23269d6e2031129b053fdb4e5a7b0637fc6c9c0586737b962b2f834493"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eca7ff7a47043cf6ce5c7f45f603b09121a7cc047447744b029d1b719278eb5"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7d3f6762217c1da954de24b4a1a70515630d29f71e268ec5000afe81377642d"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b24c19e8598916d9c6f5a5437671f55ee93c212a2c4c569605dc3842b6820386"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5a6cbf216b69c7090f0fe4669501a27326c34e119068c1494f35aaf4cc683778"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:352e95ead6964974b234e16ba8a66dad102ec7bf8ac064a23f95371d8b198aab"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d6ab88c81c4040a817aa432484e838aaddf8bfd7ca70e4e615482757acb64f8"}, + {file = "ruff-0.3.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79bca3a03a759cc773fca69e0bdeac8abd1c13c31b798d5bb3c9da4a03144a9f"}, + {file = "ruff-0.3.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2700a804d5336bcffe063fd789ca2c7b02b552d2e323a336700abb8ae9e6a3f8"}, + {file = "ruff-0.3.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd66469f1a18fdb9d32e22b79f486223052ddf057dc56dea0caaf1a47bdfaf4e"}, + {file = "ruff-0.3.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45817af234605525cdf6317005923bf532514e1ea3d9270acf61ca2440691376"}, + {file = "ruff-0.3.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0da458989ce0159555ef224d5b7c24d3d2e4bf4c300b85467b08c3261c6bc6a8"}, + {file = "ruff-0.3.3-py3-none-win32.whl", hash = "sha256:f2831ec6a580a97f1ea82ea1eda0401c3cdf512cf2045fa3c85e8ef109e87de0"}, + {file = "ruff-0.3.3-py3-none-win_amd64.whl", hash = "sha256:be90bcae57c24d9f9d023b12d627e958eb55f595428bafcb7fec0791ad25ddfc"}, + {file = "ruff-0.3.3-py3-none-win_arm64.whl", hash = "sha256:0171aab5fecdc54383993389710a3d1227f2da124d76a2784a7098e818f92d61"}, + {file = "ruff-0.3.3.tar.gz", hash = "sha256:38671be06f57a2f8aba957d9f701ea889aa5736be806f18c0cd03d6ff0cbca8d"}, +] + +[[package]] +name = "setuptools" +version = "69.2.0" +requires_python = ">=3.8" +summary = "Easily download, build, install, upgrade, and uninstall Python packages" +groups = ["check"] +files = [ + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, +] diff --git a/codegen/pyproject.toml b/codegen/pyproject.toml new file mode 100644 index 000000000000..3c42454b5530 --- /dev/null +++ b/codegen/pyproject.toml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[project] +name = "codegen" +version = "0.1.0" +description = "Generate source code for Apache OpenDAL." +authors = [ + {name = "Apache OpenDAL", email = "dev@opendal.apache.org"}, +] +dependencies = [ + "pyhumps>=3.8.0", +] +requires-python = ">=3.12" +readme = "README.md" +license = {text = "Apache-2.0"} + +[tool.pdm] +distribution = true + +[tool.pdm.scripts] +check = { composite = ["ruff check", "pyright"] } +genconfig = "bin/genconfig.py" + +[tool.pdm.dev-dependencies] +check = [ + "ruff>=0.3.3", + "pyright>=1.1.354", +] +bin = [ + "Jinja2>=3.1.3", +] + +[tool.ruff] +line-length = 120 +lint.select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "PGH", # pygrep-hooks + "RUF", # ruff + "W", # pycodestyle + "YTT", # flake8-2020 +] +src = ["src"] + +[tool.pyright] +strict = ["src", "bin"] diff --git a/codegen/src/codegen/__init__.py b/codegen/src/codegen/__init__.py new file mode 100644 index 000000000000..245692337bc3 --- /dev/null +++ b/codegen/src/codegen/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/codegen/src/codegen/config/__init__.py b/codegen/src/codegen/config/__init__.py new file mode 100644 index 000000000000..4e866082e15b --- /dev/null +++ b/codegen/src/codegen/config/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .s3 import S3 as S3 diff --git a/codegen/src/codegen/config/config.py b/codegen/src/codegen/config/config.py new file mode 100644 index 000000000000..0e79263d62d9 --- /dev/null +++ b/codegen/src/codegen/config/config.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import dataclasses +import enum +import textwrap +from typing import List + +import humps + + +@enum.unique +class FieldType(enum.Enum): + Nothing = enum.auto() + Bool = enum.auto() + Int = enum.auto() + Long = enum.auto() + Float = enum.auto() + Double = enum.auto() + String = enum.auto() + + +@enum.unique +class RefinedFieldType(enum.Enum): + Nothing = enum.auto() + RustUsize = enum.auto() + + +@dataclasses.dataclass +class ConfigField: + name: str + desc: str + + ty: FieldType = FieldType.Nothing + refined_ty: RefinedFieldType = RefinedFieldType.Nothing + + required: bool = False + sensitive: bool = False + + # default value for the config field if not set + default: str = "" + # a list of available values for the config field + available: List[str] = dataclasses.field(default_factory=list) + # a list of example values for the config field + example: List[str] = dataclasses.field(default_factory=list) + + ########### + # RustGen # + ########### + + def rust_type(self) -> str: + ty = self._rust_type() + if self.required: + return ty + return f"Option<{ty}>" + + def _rust_type(self) -> str: + match self.refined_ty: + case RefinedFieldType.Nothing: + pass + case RefinedFieldType.RustUsize: + return "usize" + match self.ty: + case FieldType.Nothing: + raise ValueError("Field type is not set") + case FieldType.Bool: + return "bool" + case FieldType.Int: + return "i32" + case FieldType.Long: + return "i64" + case FieldType.Float: + return "f32" + case FieldType.Double: + return "f64" + case FieldType.String: + return "String" + raise ValueError(f"Unknown field type: ({self.ty}, {self.refined_ty})") + + def rust_comment(self) -> str: + res = "" + + for line in textwrap.dedent(self.desc).strip().splitlines(): + if line: + res += f"/// {line}\n" + else: + res += "///\n" + + if self.default: + res += "///\n" + res += f"/// Default to `{self.default}` if not set.\n" + + if self.required: + res += "///\n" + res += "/// Required.\n" + + if self.example: + res += "///\n" + res += "/// For examples:\n" + for example in self.example: + res += f"/// - {example}\n" + + if self.available: + res += "///\n" + res += "/// Available values:\n" + for available in self.available: + res += f"/// - {available}\n" + + return res + + def rust_debug_field(self) -> str: + if not self.sensitive: + return f"&self.{self.name}" + elif self.required: + return f"desensitize_secret(&self.{self.name})" + else: + return f"&self.{self.name}.as_deref().map(desensitize_secret)" + + +@dataclasses.dataclass +class Config: + name: str + desc: str + fields: List[ConfigField] = dataclasses.field(default_factory=list) + + # Rust's #[cfg(...)] conditions for the config + rust_cfg: str = "" + + ########### + # RustGen # + ########### + + def rust_struct_name(self) -> str: + return f"{humps.pascalize(self.name)}Config" diff --git a/codegen/src/codegen/config/s3.py b/codegen/src/codegen/config/s3.py new file mode 100644 index 000000000000..318b1e72f80f --- /dev/null +++ b/codegen/src/codegen/config/s3.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .config import Config, ConfigField, FieldType, RefinedFieldType + +S3 = Config( + name="s3", + desc="AWS S3 and compatible services support (MinIO, DigitalOcean Spaces, Tencent Cloud Object Storage, etc.)", + + rust_cfg='#[cfg(feature = "services-s3")]', + + fields=[ + ConfigField( + name="root", + ty=FieldType.String, + desc=""" + root of this backend. + + All operations will happen under this root. + """, + default="/", + ), + ConfigField( + name="bucket", + ty=FieldType.String, + desc="bucket name of this backend.", + required=True, + ), + ConfigField( + name="endpoint", + ty=FieldType.String, + desc=""" + endpoint of this backend. + + Endpoint must be full uri, e.g. + + - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + - Cloudflare R2: `https://.r2.cloudflarestorage.com` + - Aliyun OSS: `https://{region}.aliyuncs.com` + - Tencent COS: `https://cos.{region}.myqcloud.com` + - Minio: `http://127.0.0.1:9000` + + If user inputs endpoint without scheme like "s3.amazonaws.com", we + will prepend "https://" before it. + """, + default="https://s3.amazonaws.com", + ), + ConfigField( + name="region", + ty=FieldType.String, + desc=""" + Region represent the signing region of this endpoint. This is required + if you are using the default AWS S3 endpoint. + + If using a custom endpoint, + - If region is set, we will take user's input first. + - If not, we will try to load it from environment. + """, + ), + ConfigField( + name="access_key_id", + ty=FieldType.String, + sensitive=True, + desc=""" + access_key_id of this backend. + + - If access_key_id is set, we will take user's input first. + - If not, we will try to load it from environment. + """, + ), + ConfigField( + name="secret_access_key", + ty=FieldType.String, + sensitive=True, + desc=""" + secret_access_key of this backend. + + - If secret_access_key is set, we will take user's input first. + - If not, we will try to load it from environment. + """, + ), + ConfigField( + name="security_token", + ty=FieldType.String, + sensitive=True, + desc=""" + security_token (aka, session token) of this backend. + + This token will expire after sometime, it's recommended to set security_token by hand. + """, + ), + ConfigField( + name="role_arn", + ty=FieldType.String, + desc=""" + role_arn for this backend. + + If `role_arn` is set, we will use already known config as source + credential to assume role with `role_arn`. + """, + ), + ConfigField( + name="external_id", + ty=FieldType.String, + desc="external_id for this backend.", + ), + ConfigField( + name="disable_config_load", + ty=FieldType.Bool, + required=True, + desc=""" + Disable config load so that opendal will not load config from environment, + e.g, envs like `AWS_ACCESS_KEY_ID` or files like `~/.aws/config` + """, + ), + ConfigField( + name="disable_ec2_metadata", + ty=FieldType.Bool, + required=True, + desc=""" + Disable load credential from ec2 metadata. + + This option is used to disable the default behavior of opendal + to load credential from ec2 metadata, a.k.a, IMDSv2 + """, + ), + ConfigField( + name="allow_anonymous", + ty=FieldType.Bool, + required=True, + desc=""" + Allow anonymous will allow opendal to send request without signing + when credential is not loaded. + """, + ), + ConfigField( + name="server_side_encryption", + ty=FieldType.String, + desc="server_side_encryption for this backend.", + available=["`AES256`", "`aws:kms`"], + ), + ConfigField( + name="server_side_encryption_aws_kms_key_id", + ty=FieldType.String, + desc=""" + server_side_encryption_aws_kms_key_id for this backend + + - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + is not set, S3 will use aws managed kms key to encrypt data. + - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + is a valid kms key id, S3 will use the provided kms key to encrypt data. + - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be + returned. + - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` + is a noop. + """, + ), + ConfigField( + name="server_side_encryption_customer_algorithm", + ty=FieldType.String, + desc="server_side_encryption_customer_algorithm for this backend.", + available=["`AES256`"] + ), + ConfigField( + name="server_side_encryption_customer_key", + ty=FieldType.String, + sensitive=True, + desc=""" + server_side_encryption_customer_key for this backend. + + # Value + + base64 encoded key that matches algorithm specified in + `server_side_encryption_customer_algorithm`. + """, + ), + ConfigField( + name="server_side_encryption_customer_key_md5", + ty=FieldType.String, + sensitive=True, + desc=""" + Set server_side_encryption_customer_key_md5 for this backend. + + # Value + + MD5 digest of key specified in `server_side_encryption_customer_key`. + """, + ), + ConfigField( + name="default_storage_class", + ty=FieldType.String, + desc=""" + default_storage_class for this backend. + + S3 compatible services don't support all of available values. + """, + available=[ + "`DEEP_ARCHIVE`", + "`GLACIER`", + "`GLACIER_IR`", + "`INTELLIGENT_TIERING`", + "`ONEZONE_IA`", + "`OUTPOSTS`", + "`REDUCED_REDUNDANCY`", + "`STANDARD`", + "`STANDARD_IA`", + ] + ), + ConfigField( + name="enable_virtual_host_style", + ty=FieldType.Bool, + required=True, + desc=""" + Enable virtual host style so that opendal will send API requests + in virtual host style instead of path style. + + - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` + - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` + """, + ), + ConfigField( + name="batch_max_operations", + ty=FieldType.Int, + refined_ty=RefinedFieldType.RustUsize, + desc=""" + Set maximum batch operations of this backend. + + Some compatible services have a limit on the number of operations in a batch request. + For example, R2 could return `Internal Error` while batch delete 1000 files. + + Please tune this value based on services' document. + """, + ), + ConfigField( + name="disable_stat_with_override", + ty=FieldType.Bool, + required=True, + desc=""" + Disable stat with override so that opendal will not send stat request with override queries. + + For example, R2 doesn't support stat with `response_content_type` query. + """, + ) + ] +) diff --git a/codegen/template/config.rs.j2 b/codegen/template/config.rs.j2 new file mode 100644 index 000000000000..4ed7ce160824 --- /dev/null +++ b/codegen/template/config.rs.j2 @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This file is generated by codegen/bin/genconfig.py. +// DO NOT MODIFY MANUALLY + +{% for config in configs %} +{{ config.rust_cfg }} +pub use {{ config.name }}::{{ config.rust_struct_name() }}; +{% endfor %} + +{% for config in configs %} +{{ config.rust_cfg }} +pub mod {{ config.name }} { + use crate::raw::*; + use serde::Deserialize; + use std::fmt::Debug; + use std::fmt::Formatter; + + /// Config for {{ config.desc }} + #[derive(Default, Deserialize)] + #[serde(default)] + #[non_exhaustive] + pub struct {{ config.rust_struct_name() }} { + {%- for field in config.fields %} + {{ field.rust_comment() -}} + pub {{ field.name }}: {{ field.rust_type() }}, + {%- endfor %} + } + + impl Debug for {{ config.rust_struct_name() }} { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("{{ config.rust_struct_name() }}"); + + d + {% for field in config.fields %} + .field("{{ field.name }}", {{ field.rust_debug_field() }}) + {% endfor %} + ; + + d.finish_non_exhaustive() + } + } +} +{% endfor %} diff --git a/core/src/raw/mod.rs b/core/src/raw/mod.rs index e58d6e9ced70..3bf571efd32a 100644 --- a/core/src/raw/mod.rs +++ b/core/src/raw/mod.rs @@ -77,6 +77,9 @@ pub use futures_util::ConcurrentFutures; mod enum_utils; pub use enum_utils::*; +mod service_util; +pub use service_util::*; + // Expose as a pub mod to avoid confusing. pub mod adapters; pub mod oio; diff --git a/core/src/raw/service_util.rs b/core/src/raw/service_util.rs new file mode 100644 index 000000000000..55b79e2f98b5 --- /dev/null +++ b/core/src/raw/service_util.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// A helper function to desensitize secret string. +pub fn desensitize_secret(s: &str) -> String { + // Always use 3 stars to mask the secret if length is low. + // + // # NOTE + // + // It's by design to use 10 instead of 6. Attackers could brute force the secrets + // if the length is too short. + if s.len() <= 10 { + return "***".to_string(); + } + + // Keep the first & end three chars visible for easier debugging. + format!("{}***{}", &s[..3], &s[s.len() - 3..]) +} diff --git a/core/src/services/config.rs b/core/src/services/config.rs new file mode 100644 index 000000000000..d26dfca7014a --- /dev/null +++ b/core/src/services/config.rs @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This file is generated by codegen/bin/genconfig.py. +// DO NOT MODIFY MANUALLY + +#[cfg(feature = "services-s3")] +pub use s3::S3Config; + +#[cfg(feature = "services-s3")] +pub mod s3 { + use crate::raw::*; + use serde::Deserialize; + use std::fmt::Debug; + use std::fmt::Formatter; + + /// Config for AWS S3 and compatible services support (MinIO, DigitalOcean Spaces, Tencent Cloud Object Storage, etc.) + #[derive(Default, Deserialize)] + #[serde(default)] + #[non_exhaustive] + pub struct S3Config { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// Default to `/` if not set. + pub root: Option, + /// bucket name of this backend. + /// + /// Required. + pub bucket: String, + /// endpoint of this backend. + /// + /// Endpoint must be full uri, e.g. + /// + /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` + /// - Aliyun OSS: `https://{region}.aliyuncs.com` + /// - Tencent COS: `https://cos.{region}.myqcloud.com` + /// - Minio: `http://127.0.0.1:9000` + /// + /// If user inputs endpoint without scheme like "s3.amazonaws.com", we + /// will prepend "https://" before it. + /// + /// Default to `https://s3.amazonaws.com` if not set. + pub endpoint: Option, + /// Region represent the signing region of this endpoint. This is required + /// if you are using the default AWS S3 endpoint. + /// + /// If using a custom endpoint, + /// - If region is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub region: Option, + /// access_key_id of this backend. + /// + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub access_key_id: Option, + /// secret_access_key of this backend. + /// + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub secret_access_key: Option, + /// security_token (aka, session token) of this backend. + /// + /// This token will expire after sometime, it's recommended to set security_token by hand. + pub security_token: Option, + /// role_arn for this backend. + /// + /// If `role_arn` is set, we will use already known config as source + /// credential to assume role with `role_arn`. + pub role_arn: Option, + /// external_id for this backend. + pub external_id: Option, + /// Disable config load so that opendal will not load config from environment, + /// e.g, envs like `AWS_ACCESS_KEY_ID` or files like `~/.aws/config` + /// + /// Required. + pub disable_config_load: bool, + /// Disable load credential from ec2 metadata. + /// + /// This option is used to disable the default behavior of opendal + /// to load credential from ec2 metadata, a.k.a, IMDSv2 + /// + /// Required. + pub disable_ec2_metadata: bool, + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + /// + /// Required. + pub allow_anonymous: bool, + /// server_side_encryption for this backend. + /// + /// Available values: + /// - `AES256` + /// - `aws:kms` + pub server_side_encryption: Option, + /// server_side_encryption_aws_kms_key_id for this backend + /// + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is not set, S3 will use aws managed kms key to encrypt data. + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is a valid kms key id, S3 will use the provided kms key to encrypt data. + /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be + /// returned. + /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` + /// is a noop. + pub server_side_encryption_aws_kms_key_id: Option, + /// server_side_encryption_customer_algorithm for this backend. + /// + /// Available values: + /// - `AES256` + pub server_side_encryption_customer_algorithm: Option, + /// server_side_encryption_customer_key for this backend. + /// + /// # Value + /// + /// base64 encoded key that matches algorithm specified in + /// `server_side_encryption_customer_algorithm`. + pub server_side_encryption_customer_key: Option, + /// Set server_side_encryption_customer_key_md5 for this backend. + /// + /// # Value + /// + /// MD5 digest of key specified in `server_side_encryption_customer_key`. + pub server_side_encryption_customer_key_md5: Option, + /// default_storage_class for this backend. + /// + /// S3 compatible services don't support all of available values. + /// + /// Available values: + /// - `DEEP_ARCHIVE` + /// - `GLACIER` + /// - `GLACIER_IR` + /// - `INTELLIGENT_TIERING` + /// - `ONEZONE_IA` + /// - `OUTPOSTS` + /// - `REDUCED_REDUNDANCY` + /// - `STANDARD` + /// - `STANDARD_IA` + pub default_storage_class: Option, + /// Enable virtual host style so that opendal will send API requests + /// in virtual host style instead of path style. + /// + /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` + /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` + /// + /// Required. + pub enable_virtual_host_style: bool, + /// Set maximum batch operations of this backend. + /// + /// Some compatible services have a limit on the number of operations in a batch request. + /// For example, R2 could return `Internal Error` while batch delete 1000 files. + /// + /// Please tune this value based on services' document. + pub batch_max_operations: Option, + /// Disable stat with override so that opendal will not send stat request with override queries. + /// + /// For example, R2 doesn't support stat with `response_content_type` query. + /// + /// Required. + pub disable_stat_with_override: bool, + } + + impl Debug for S3Config { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("S3Config"); + + d.field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field( + "access_key_id", + &self.access_key_id.as_deref().map(desensitize_secret), + ) + .field( + "secret_access_key", + &self.secret_access_key.as_deref().map(desensitize_secret), + ) + .field( + "security_token", + &self.security_token.as_deref().map(desensitize_secret), + ) + .field("role_arn", &self.role_arn) + .field("external_id", &self.external_id) + .field("disable_config_load", &self.disable_config_load) + .field("disable_ec2_metadata", &self.disable_ec2_metadata) + .field("allow_anonymous", &self.allow_anonymous) + .field("server_side_encryption", &self.server_side_encryption) + .field( + "server_side_encryption_aws_kms_key_id", + &self.server_side_encryption_aws_kms_key_id, + ) + .field( + "server_side_encryption_customer_algorithm", + &self.server_side_encryption_customer_algorithm, + ) + .field( + "server_side_encryption_customer_key", + &self + .server_side_encryption_customer_key + .as_deref() + .map(desensitize_secret), + ) + .field( + "server_side_encryption_customer_key_md5", + &self + .server_side_encryption_customer_key_md5 + .as_deref() + .map(desensitize_secret), + ) + .field("default_storage_class", &self.default_storage_class) + .field("enable_virtual_host_style", &self.enable_virtual_host_style) + .field("batch_max_operations", &self.batch_max_operations) + .field( + "disable_stat_with_override", + &self.disable_stat_with_override, + ); + + d.finish_non_exhaustive() + } + } +} diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index 71cd71b8084f..c0f281197c93 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -19,6 +19,8 @@ //! //! More ongoing services support is tracked at [opendal#5](https://github.com/apache/opendal/issues/5). Please feel free to submit issues if there are services not covered. +mod config; + #[cfg(feature = "services-azblob")] mod azblob; #[cfg(feature = "services-azblob")] diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index bb7b3949c383..f2fcab13e265 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -45,6 +45,7 @@ use super::error::parse_s3_error_code; use super::lister::S3Lister; use super::writer::S3Writer; use super::writer::S3Writers; +use super::S3Config; use crate::raw::*; use crate::*; @@ -61,161 +62,6 @@ static ENDPOINT_TEMPLATES: Lazy> = Lazy::new const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; -/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. -#[derive(Default, Deserialize)] -#[serde(default)] -#[non_exhaustive] -pub struct S3Config { - /// root of this backend. - /// - /// All operations will happen under this root. - /// - /// default to `/` if not set. - pub root: Option, - /// bucket name of this backend. - /// - /// required. - pub bucket: String, - /// endpoint of this backend. - /// - /// Endpoint must be full uri, e.g. - /// - /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` - /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` - /// - Aliyun OSS: `https://{region}.aliyuncs.com` - /// - Tencent COS: `https://cos.{region}.myqcloud.com` - /// - Minio: `http://127.0.0.1:9000` - /// - /// If user inputs endpoint without scheme like "s3.amazonaws.com", we - /// will prepend "https://" before it. - /// - /// default to `https://s3.amazonaws.com` if not set. - pub endpoint: Option, - /// Region represent the signing region of this endpoint. This is required - /// if you are using the default AWS S3 endpoint. - /// - /// If using a custom endpoint, - /// - If region is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub region: Option, - - /// access_key_id of this backend. - /// - /// - If access_key_id is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub access_key_id: Option, - /// secret_access_key of this backend. - /// - /// - If secret_access_key is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub secret_access_key: Option, - /// security_token (aka, session token) of this backend. - /// - /// This token will expire after sometime, it's recommended to set security_token - /// by hand. - pub security_token: Option, - /// role_arn for this backend. - /// - /// If `role_arn` is set, we will use already known config as source - /// credential to assume role with `role_arn`. - pub role_arn: Option, - /// external_id for this backend. - pub external_id: Option, - /// Disable config load so that opendal will not load config from - /// environment. - /// - /// For examples: - /// - /// - envs like `AWS_ACCESS_KEY_ID` - /// - files like `~/.aws/config` - pub disable_config_load: bool, - /// Disable load credential from ec2 metadata. - /// - /// This option is used to disable the default behavior of opendal - /// to load credential from ec2 metadata, a.k.a, IMDSv2 - pub disable_ec2_metadata: bool, - /// Allow anonymous will allow opendal to send request without signing - /// when credential is not loaded. - pub allow_anonymous: bool, - /// server_side_encryption for this backend. - /// - /// Available values: `AES256`, `aws:kms`. - pub server_side_encryption: Option, - /// server_side_encryption_aws_kms_key_id for this backend - /// - /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` - /// is not set, S3 will use aws managed kms key to encrypt data. - /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` - /// is a valid kms key id, S3 will use the provided kms key to encrypt data. - /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be - /// returned. - /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` - /// is a noop. - pub server_side_encryption_aws_kms_key_id: Option, - /// server_side_encryption_customer_algorithm for this backend. - /// - /// Available values: `AES256`. - pub server_side_encryption_customer_algorithm: Option, - /// server_side_encryption_customer_key for this backend. - /// - /// # Value - /// - /// base64 encoded key that matches algorithm specified in - /// `server_side_encryption_customer_algorithm`. - pub server_side_encryption_customer_key: Option, - /// Set server_side_encryption_customer_key_md5 for this backend. - /// - /// # Value - /// - /// MD5 digest of key specified in `server_side_encryption_customer_key`. - pub server_side_encryption_customer_key_md5: Option, - /// default storage_class for this backend. - /// - /// Available values: - /// - `DEEP_ARCHIVE` - /// - `GLACIER` - /// - `GLACIER_IR` - /// - `INTELLIGENT_TIERING` - /// - `ONEZONE_IA` - /// - `OUTPOSTS` - /// - `REDUCED_REDUNDANCY` - /// - `STANDARD` - /// - `STANDARD_IA` - /// - /// S3 compatible services don't support all of them - pub default_storage_class: Option, - /// Enable virtual host style so that opendal will send API requests - /// in virtual host style instead of path style. - /// - /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` - /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` - pub enable_virtual_host_style: bool, - /// Set maximum batch operations of this backend. - /// - /// Some compatible services have a limit on the number of operations in a batch request. - /// For example, R2 could return `Internal Error` while batch delete 1000 files. - /// - /// Please tune this value based on services' document. - pub batch_max_operations: Option, - /// Disable stat with override so that opendal will not send stat request with override queries. - /// - /// For example, R2 doesn't support stat with `response_content_type` query. - pub disable_stat_with_override: bool, -} - -impl Debug for S3Config { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("S3Config"); - - d.field("root", &self.root) - .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint) - .field("region", &self.region); - - d.finish_non_exhaustive() - } -} - /// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. /// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services). #[doc = include_str!("docs.md")] diff --git a/core/src/services/s3/mod.rs b/core/src/services/s3/mod.rs index 4e2f283cb215..acb580f13199 100644 --- a/core/src/services/s3/mod.rs +++ b/core/src/services/s3/mod.rs @@ -16,8 +16,8 @@ // under the License. mod backend; +pub use super::config::S3Config; pub use backend::S3Builder as S3; -pub use backend::S3Config; mod core; mod error; diff --git a/licenserc.toml b/licenserc.toml index b36ac6795029..c930478e6bf4 100644 --- a/licenserc.toml +++ b/licenserc.toml @@ -29,7 +29,8 @@ excludes = [ "**/DEPENDENCIES.*.tsv", "**/DEPENDENCIES.*.csv", - # Python binding related files + # Python virtual env files + "**/.venv/**", "**/venv/**", # Website generated files