diff --git a/core/src/services/aliyun_drive/backend.rs b/core/src/services/aliyun_drive/backend.rs index 7dc5ba5c21cd..86a1f6a21b34 100644 --- a/core/src/services/aliyun_drive/backend.rs +++ b/core/src/services/aliyun_drive/backend.rs @@ -26,8 +26,6 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::Mutex; use super::core::*; @@ -36,58 +34,9 @@ use super::lister::AliyunDriveLister; use super::lister::AliyunDriveParent; use super::writer::AliyunDriveWriter; use crate::raw::*; +use crate::services::AliyunDriveConfig; use crate::*; -/// Aliyun Drive services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct AliyunDriveConfig { - /// The Root of this backend. - /// - /// All operations will happen under this root. - /// - /// Default to `/` if not set. - pub root: Option, - /// The access_token of this backend. - /// - /// Solution for client-only purpose. #4733 - /// - /// Required if no client_id, client_secret and refresh_token are provided. - pub access_token: Option, - /// The client_id of this backend. - /// - /// Required if no access_token is provided. - pub client_id: Option, - /// The client_secret of this backend. - /// - /// Required if no access_token is provided. - pub client_secret: Option, - /// The refresh_token of this backend. - /// - /// Required if no access_token is provided. - pub refresh_token: Option, - /// The drive_type of this backend. - /// - /// All operations will happen under this type of drive. - /// - /// Available values are `default`, `backup` and `resource`. - /// - /// Fallback to default if not set or no other drives can be found. - pub drive_type: String, -} - -impl Debug for AliyunDriveConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("AliyunDriveConfig"); - - d.field("root", &self.root) - .field("drive_type", &self.drive_type); - - d.finish_non_exhaustive() - } -} - impl Configurator for AliyunDriveConfig { type Builder = AliyunDriveBuilder; fn into_builder(self) -> Self::Builder { @@ -205,12 +154,12 @@ impl Builder for AliyunDriveBuilder { self.config.refresh_token.clone(), ) { (Some(client_id), Some(client_secret), Some(refresh_token)) if - !client_id.is_empty() && !client_secret.is_empty() && !refresh_token.is_empty() => { + !client_id.is_empty() && !client_secret.is_empty() && !refresh_token.is_empty() => { AliyunDriveSign::Refresh(client_id, client_secret, refresh_token, None, 0) } _ => return Err(Error::new( - ErrorKind::ConfigInvalid, - "access_token and a set of client_id, client_secret, and refresh_token are both missing.") + ErrorKind::ConfigInvalid, + "access_token and a set of client_id, client_secret, and refresh_token are both missing.") .with_operation("Builder::build") .with_context("service", Scheme::AliyunDrive)), }, diff --git a/core/src/services/aliyun_drive/config.rs b/core/src/services/aliyun_drive/config.rs new file mode 100644 index 000000000000..f03a650ced83 --- /dev/null +++ b/core/src/services/aliyun_drive/config.rs @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Aliyun Drive services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct AliyunDriveConfig { + /// The Root of this backend. + /// + /// All operations will happen under this root. + /// + /// Default to `/` if not set. + pub root: Option, + /// The access_token of this backend. + /// + /// Solution for client-only purpose. #4733 + /// + /// Required if no client_id, client_secret and refresh_token are provided. + pub access_token: Option, + /// The client_id of this backend. + /// + /// Required if no access_token is provided. + pub client_id: Option, + /// The client_secret of this backend. + /// + /// Required if no access_token is provided. + pub client_secret: Option, + /// The refresh_token of this backend. + /// + /// Required if no access_token is provided. + pub refresh_token: Option, + /// The drive_type of this backend. + /// + /// All operations will happen under this type of drive. + /// + /// Available values are `default`, `backup` and `resource`. + /// + /// Fallback to default if not set or no other drives can be found. + pub drive_type: String, +} + +impl Debug for AliyunDriveConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("AliyunDriveConfig"); + + d.field("root", &self.root) + .field("drive_type", &self.drive_type); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/aliyun_drive/mod.rs b/core/src/services/aliyun_drive/mod.rs index efdc2b8eefbe..5481bf4fbe9a 100644 --- a/core/src/services/aliyun_drive/mod.rs +++ b/core/src/services/aliyun_drive/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-aliyun-drive")] +mod core; + +#[cfg(feature = "services-aliyun-drive")] mod backend; +#[cfg(feature = "services-aliyun-drive")] mod error; +#[cfg(feature = "services-aliyun-drive")] mod lister; +#[cfg(feature = "services-aliyun-drive")] mod writer; +#[cfg(feature = "services-aliyun-drive")] pub use backend::AliyunDriveBuilder as AliyunDrive; -pub use backend::AliyunDriveConfig; -mod core; +mod config; +pub use config::AliyunDriveConfig; diff --git a/core/src/services/alluxio/backend.rs b/core/src/services/alluxio/backend.rs index bd998430b157..8e8bdc148366 100644 --- a/core/src/services/alluxio/backend.rs +++ b/core/src/services/alluxio/backend.rs @@ -21,8 +21,6 @@ use std::sync::Arc; use http::Response; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::AlluxioCore; use super::error::parse_error; @@ -30,36 +28,9 @@ use super::lister::AlluxioLister; use super::writer::AlluxioWriter; use super::writer::AlluxioWriters; use crate::raw::*; +use crate::services::AlluxioConfig; use crate::*; -/// Config for alluxio services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct AlluxioConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - /// - /// default to `/` if not set. - pub root: Option, - /// endpoint of this backend. - /// - /// Endpoint must be full uri, mostly like `http://127.0.0.1:39999`. - pub endpoint: Option, -} - -impl Debug for AlluxioConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("AlluxioConfig"); - - d.field("root", &self.root) - .field("endpoint", &self.endpoint); - - d.finish_non_exhaustive() - } -} - impl Configurator for AlluxioConfig { type Builder = AlluxioBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/alluxio/config.rs b/core/src/services/alluxio/config.rs new file mode 100644 index 000000000000..78f143cfc28d --- /dev/null +++ b/core/src/services/alluxio/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for alluxio services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct AlluxioConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// default to `/` if not set. + pub root: Option, + /// endpoint of this backend. + /// + /// Endpoint must be full uri, mostly like `http://127.0.0.1:39999`. + pub endpoint: Option, +} + +impl Debug for AlluxioConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("AlluxioConfig"); + + d.field("root", &self.root) + .field("endpoint", &self.endpoint); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/alluxio/mod.rs b/core/src/services/alluxio/mod.rs index 95e0216d34b0..aa2c88b6fe35 100644 --- a/core/src/services/alluxio/mod.rs +++ b/core/src/services/alluxio/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::AlluxioBuilder as Alluxio; -pub use backend::AlluxioConfig; - +#[cfg(feature = "services-alluxio")] mod core; +#[cfg(feature = "services-alluxio")] mod error; +#[cfg(feature = "services-alluxio")] mod lister; +#[cfg(feature = "services-alluxio")] mod writer; + +#[cfg(feature = "services-alluxio")] +mod backend; +#[cfg(feature = "services-alluxio")] +pub use backend::AlluxioBuilder as Alluxio; + +mod config; +pub use config::AlluxioConfig; diff --git a/core/src/services/atomicserver/backend.rs b/core/src/services/atomicserver/backend.rs index e6e415674ba7..73f51994375f 100644 --- a/core/src/services/atomicserver/backend.rs +++ b/core/src/services/atomicserver/backend.rs @@ -30,38 +30,9 @@ use serde::Serialize; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::AtomicserverConfig; use crate::*; -/// Atomicserver service support. - -/// Config for Atomicserver services support -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct AtomicserverConfig { - /// work dir of this backend - pub root: Option, - /// endpoint of this backend - pub endpoint: Option, - /// private_key of this backend - pub private_key: Option, - /// public_key of this backend - pub public_key: Option, - /// parent_resource_id of this backend - pub parent_resource_id: Option, -} - -impl Debug for AtomicserverConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AtomicserverConfig") - .field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("public_key", &self.public_key) - .field("parent_resource_id", &self.parent_resource_id) - .finish_non_exhaustive() - } -} - impl Configurator for AtomicserverConfig { type Builder = AtomicserverBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/atomicserver/config.rs b/core/src/services/atomicserver/config.rs new file mode 100644 index 000000000000..9b44ce8196cc --- /dev/null +++ b/core/src/services/atomicserver/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Atomicserver services support +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct AtomicserverConfig { + /// work dir of this backend + pub root: Option, + /// endpoint of this backend + pub endpoint: Option, + /// private_key of this backend + pub private_key: Option, + /// public_key of this backend + pub public_key: Option, + /// parent_resource_id of this backend + pub parent_resource_id: Option, +} + +impl Debug for AtomicserverConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AtomicserverConfig") + .field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("public_key", &self.public_key) + .field("parent_resource_id", &self.parent_resource_id) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/atomicserver/mod.rs b/core/src/services/atomicserver/mod.rs index 45b4185640c2..b1e3ea4f3974 100644 --- a/core/src/services/atomicserver/mod.rs +++ b/core/src/services/atomicserver/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-atomicserver")] mod backend; - +#[cfg(feature = "services-atomicserver")] pub use backend::AtomicserverBuilder as Atomicserver; -pub use backend::AtomicserverConfig; + +mod config; +pub use config::AtomicserverConfig; diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index ad59f5254185..c084223cc00b 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -30,8 +30,6 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; -use serde::Serialize; use sha2::Digest; use sha2::Sha256; @@ -41,6 +39,7 @@ use super::writer::AzblobWriter; use crate::raw::*; use crate::services::azblob::core::AzblobCore; use crate::services::azblob::writer::AzblobWriters; +use crate::services::AzblobConfig; use crate::*; /// Known endpoint suffix Azure Storage Blob services resource URI syntax. @@ -55,69 +54,6 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[ const AZBLOB_BATCH_LIMIT: usize = 256; -/// Azure Storage Blob services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct AzblobConfig { - /// The root of Azblob service backend. - /// - /// All operations will happen under this root. - pub root: Option, - - /// The container name of Azblob service backend. - pub container: String, - - /// The endpoint of Azblob service backend. - /// - /// Endpoint must be full uri, e.g. - /// - /// - Azblob: `https://accountname.blob.core.windows.net` - /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1` - pub endpoint: Option, - - /// The account name of Azblob service backend. - pub account_name: Option, - - /// The account key of Azblob service backend. - pub account_key: Option, - - /// The encryption key of Azblob service backend. - pub encryption_key: Option, - - /// The encryption key sha256 of Azblob service backend. - pub encryption_key_sha256: Option, - - /// The encryption algorithm of Azblob service backend. - pub encryption_algorithm: Option, - - /// The sas token of Azblob service backend. - pub sas_token: Option, - - /// The maximum batch operations of Azblob service backend. - pub batch_max_operations: Option, -} - -impl Debug for AzblobConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("AzblobConfig"); - - ds.field("root", &self.root); - ds.field("container", &self.container); - ds.field("endpoint", &self.endpoint); - - if self.account_name.is_some() { - ds.field("account_name", &""); - } - if self.account_key.is_some() { - ds.field("account_key", &""); - } - if self.sas_token.is_some() { - ds.field("sas_token", &""); - } - - ds.finish() - } -} - impl Configurator for AzblobConfig { type Builder = AzblobBuilder; fn into_builder(self) -> Self::Builder { @@ -840,7 +776,7 @@ TableEndpoint=http://127.0.0.1:10002/devstoreaccount1; SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D "#, ) - .expect("from connection string must succeed"); + .expect("from connection string must succeed"); assert_eq!( builder.config.endpoint.unwrap(), @@ -861,7 +797,7 @@ AccountKey=account-key; SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D "#, ) - .expect("from connection string must succeed"); + .expect("from connection string must succeed"); // SAS should be preferred over shared key assert_eq!(builder.config.sas_token.unwrap(), "sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D"); diff --git a/core/src/services/azblob/config.rs b/core/src/services/azblob/config.rs new file mode 100644 index 000000000000..546ff1e832c4 --- /dev/null +++ b/core/src/services/azblob/config.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Azure Storage Blob services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct AzblobConfig { + /// The root of Azblob service backend. + /// + /// All operations will happen under this root. + pub root: Option, + + /// The container name of Azblob service backend. + pub container: String, + + /// The endpoint of Azblob service backend. + /// + /// Endpoint must be full uri, e.g. + /// + /// - Azblob: `https://accountname.blob.core.windows.net` + /// - Azurite: `http://127.0.0.1:10000/devstoreaccount1` + pub endpoint: Option, + + /// The account name of Azblob service backend. + pub account_name: Option, + + /// The account key of Azblob service backend. + pub account_key: Option, + + /// The encryption key of Azblob service backend. + pub encryption_key: Option, + + /// The encryption key sha256 of Azblob service backend. + pub encryption_key_sha256: Option, + + /// The encryption algorithm of Azblob service backend. + pub encryption_algorithm: Option, + + /// The sas token of Azblob service backend. + pub sas_token: Option, + + /// The maximum batch operations of Azblob service backend. + pub batch_max_operations: Option, +} + +impl Debug for AzblobConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzblobConfig"); + + ds.field("root", &self.root); + ds.field("container", &self.container); + ds.field("endpoint", &self.endpoint); + + if self.account_name.is_some() { + ds.field("account_name", &""); + } + if self.account_key.is_some() { + ds.field("account_key", &""); + } + if self.sas_token.is_some() { + ds.field("sas_token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs index da314f5b3ea8..ba65d343939e 100644 --- a/core/src/services/azblob/mod.rs +++ b/core/src/services/azblob/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::AzblobBuilder as Azblob; -pub use backend::AzblobConfig; - +#[cfg(feature = "services-azblob")] mod core; +#[cfg(feature = "services-azblob")] mod error; +#[cfg(feature = "services-azblob")] mod lister; +#[cfg(feature = "services-azblob")] mod writer; + +#[cfg(feature = "services-azblob")] +mod backend; +#[cfg(feature = "services-azblob")] +pub use backend::AzblobBuilder as Azblob; + +mod config; +pub use config::AzblobConfig; diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index 0af18e2819df..2c5bd72feab7 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -25,8 +25,6 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; -use serde::Serialize; use super::core::AzdlsCore; use super::error::parse_error; @@ -34,6 +32,7 @@ use super::lister::AzdlsLister; use super::writer::AzdlsWriter; use super::writer::AzdlsWriters; use crate::raw::*; +use crate::services::AzdlsConfig; use crate::*; /// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax. @@ -46,40 +45,6 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[ "dfs.core.chinacloudapi.cn", ]; -/// Azure Data Lake Storage Gen2 Support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct AzdlsConfig { - /// Root of this backend. - pub root: Option, - /// Filesystem name of this backend. - pub filesystem: String, - /// Endpoint of this backend. - pub endpoint: Option, - /// Account name of this backend. - pub account_name: Option, - /// Account key of this backend. - pub account_key: Option, -} - -impl Debug for AzdlsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("AzdlsConfig"); - - ds.field("root", &self.root); - ds.field("filesystem", &self.filesystem); - ds.field("endpoint", &self.endpoint); - - if self.account_name.is_some() { - ds.field("account_name", &""); - } - if self.account_key.is_some() { - ds.field("account_key", &""); - } - - ds.finish() - } -} - impl Configurator for AzdlsConfig { type Builder = AzdlsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/azdls/config.rs b/core/src/services/azdls/config.rs new file mode 100644 index 000000000000..0d299a59b4fa --- /dev/null +++ b/core/src/services/azdls/config.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Azure Data Lake Storage Gen2 Support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct AzdlsConfig { + /// Root of this backend. + pub root: Option, + /// Filesystem name of this backend. + pub filesystem: String, + /// Endpoint of this backend. + pub endpoint: Option, + /// Account name of this backend. + pub account_name: Option, + /// Account key of this backend. + pub account_key: Option, +} + +impl Debug for AzdlsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzdlsConfig"); + + ds.field("root", &self.root); + ds.field("filesystem", &self.filesystem); + ds.field("endpoint", &self.endpoint); + + if self.account_name.is_some() { + ds.field("account_name", &""); + } + if self.account_key.is_some() { + ds.field("account_key", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/azdls/mod.rs b/core/src/services/azdls/mod.rs index 911cbb1ec934..321899ccc386 100644 --- a/core/src/services/azdls/mod.rs +++ b/core/src/services/azdls/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::AzdlsBuilder as Azdls; -pub use backend::AzdlsConfig; - +#[cfg(feature = "services-azdls")] mod core; +#[cfg(feature = "services-azdls")] mod error; +#[cfg(feature = "services-azdls")] mod lister; +#[cfg(feature = "services-azdls")] mod writer; + +#[cfg(feature = "services-azdls")] +mod backend; +#[cfg(feature = "services-azdls")] +pub use backend::AzdlsBuilder as Azdls; + +mod config; +pub use config::AzdlsConfig; diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index 9f6d386b35a2..61ab64717d03 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -25,8 +25,6 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; -use serde::Deserialize; -use serde::Serialize; use super::core::AzfileCore; use super::error::parse_error; @@ -34,50 +32,12 @@ use super::writer::AzfileWriter; use super::writer::AzfileWriters; use crate::raw::*; use crate::services::azfile::lister::AzfileLister; +use crate::services::AzfileConfig; use crate::*; /// Default endpoint of Azure File services. const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net"; -/// Azure File services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct AzfileConfig { - /// The root path for azfile. - pub root: Option, - /// The endpoint for azfile. - pub endpoint: Option, - /// The share name for azfile. - pub share_name: String, - /// The account name for azfile. - pub account_name: Option, - /// The account key for azfile. - pub account_key: Option, - /// The sas token for azfile. - pub sas_token: Option, -} - -impl Debug for AzfileConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("AzfileConfig"); - - ds.field("root", &self.root); - ds.field("share_name", &self.share_name); - ds.field("endpoint", &self.endpoint); - - if self.account_name.is_some() { - ds.field("account_name", &""); - } - if self.account_key.is_some() { - ds.field("account_key", &""); - } - if self.sas_token.is_some() { - ds.field("sas_token", &""); - } - - ds.finish() - } -} - impl Configurator for AzfileConfig { type Builder = AzfileBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/azfile/config.rs b/core/src/services/azfile/config.rs new file mode 100644 index 000000000000..6f3e3c957e3c --- /dev/null +++ b/core/src/services/azfile/config.rs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Azure File services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct AzfileConfig { + /// The root path for azfile. + pub root: Option, + /// The endpoint for azfile. + pub endpoint: Option, + /// The share name for azfile. + pub share_name: String, + /// The account name for azfile. + pub account_name: Option, + /// The account key for azfile. + pub account_key: Option, + /// The sas token for azfile. + pub sas_token: Option, +} + +impl Debug for AzfileConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzfileConfig"); + + ds.field("root", &self.root); + ds.field("share_name", &self.share_name); + ds.field("endpoint", &self.endpoint); + + if self.account_name.is_some() { + ds.field("account_name", &""); + } + if self.account_key.is_some() { + ds.field("account_key", &""); + } + if self.sas_token.is_some() { + ds.field("sas_token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/azfile/mod.rs b/core/src/services/azfile/mod.rs index 1047102572f1..493f8cfb7d3b 100644 --- a/core/src/services/azfile/mod.rs +++ b/core/src/services/azfile/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -pub use backend::AzfileBuilder as Azfile; -pub use backend::AzfileConfig; - -mod backend; +#[cfg(feature = "services-azfile")] mod core; +#[cfg(feature = "services-azfile")] mod error; +#[cfg(feature = "services-azfile")] mod lister; +#[cfg(feature = "services-azfile")] mod writer; + +#[cfg(feature = "services-azfile")] +mod backend; +#[cfg(feature = "services-azfile")] +pub use backend::AzfileBuilder as Azfile; + +mod config; +pub use config::AzfileConfig; diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs index 18fb45d53d2e..348c5ad57ce3 100644 --- a/core/src/services/b2/backend.rs +++ b/core/src/services/b2/backend.rs @@ -24,8 +24,6 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::RwLock; use super::core::constants; @@ -38,50 +36,9 @@ use super::writer::B2Writers; use crate::raw::*; use crate::services::b2::core::B2Signer; use crate::services::b2::core::ListFileNamesResponse; +use crate::services::B2Config; use crate::*; -/// Config for backblaze b2 services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct B2Config { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// keyID of this backend. - /// - /// - If application_key_id is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub application_key_id: Option, - /// applicationKey of this backend. - /// - /// - If application_key is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub application_key: Option, - /// bucket of this backend. - /// - /// required. - pub bucket: String, - /// bucket id of this backend. - /// - /// required. - pub bucket_id: String, -} - -impl Debug for B2Config { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("B2Config"); - - d.field("root", &self.root) - .field("application_key_id", &self.application_key_id) - .field("bucket_id", &self.bucket_id) - .field("bucket", &self.bucket); - - d.finish_non_exhaustive() - } -} - impl Configurator for B2Config { type Builder = B2Builder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/b2/config.rs b/core/src/services/b2/config.rs new file mode 100644 index 000000000000..3bfbca072b56 --- /dev/null +++ b/core/src/services/b2/config.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for backblaze b2 services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct B2Config { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// keyID of this backend. + /// + /// - If application_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub application_key_id: Option, + /// applicationKey of this backend. + /// + /// - If application_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub application_key: Option, + /// bucket of this backend. + /// + /// required. + pub bucket: String, + /// bucket id of this backend. + /// + /// required. + pub bucket_id: String, +} + +impl Debug for B2Config { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("B2Config"); + + d.field("root", &self.root) + .field("application_key_id", &self.application_key_id) + .field("bucket_id", &self.bucket_id) + .field("bucket", &self.bucket); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/b2/mod.rs b/core/src/services/b2/mod.rs index 80745f01e82a..ec294579e4d6 100644 --- a/core/src/services/b2/mod.rs +++ b/core/src/services/b2/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::B2Builder as B2; -pub use backend::B2Config; - +#[cfg(feature = "services-b2")] mod core; +#[cfg(feature = "services-b2")] mod error; +#[cfg(feature = "services-b2")] mod lister; +#[cfg(feature = "services-b2")] mod writer; + +#[cfg(feature = "services-b2")] +mod backend; +#[cfg(feature = "services-b2")] +pub use backend::B2Builder as B2; + +mod config; +pub use config::B2Config; diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index 977127f9ca1b..85914d864fd1 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -20,24 +20,16 @@ use std::fmt::Formatter; use std::str; use cacache; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::kv; use crate::raw::Access; +use crate::services::CacacheConfig; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; -/// cacache service support. -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -pub struct CacacheConfig { - /// That path to the cacache data directory. - pub datadir: Option, -} - impl Configurator for CacacheConfig { type Builder = CacacheBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/cacache/config.rs b/core/src/services/cacache/config.rs new file mode 100644 index 000000000000..768c87539b69 --- /dev/null +++ b/core/src/services/cacache/config.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// cacache service support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct CacacheConfig { + /// That path to the cacache data directory. + pub datadir: Option, +} diff --git a/core/src/services/cacache/mod.rs b/core/src/services/cacache/mod.rs index b48d7f2b7633..33c73f972f13 100644 --- a/core/src/services/cacache/mod.rs +++ b/core/src/services/cacache/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-cacache")] mod backend; - +#[cfg(feature = "services-cacache")] pub use backend::CacacheBuilder as Cacache; -pub use backend::CacacheConfig; + +mod config; +pub use config::CacacheConfig; diff --git a/core/src/services/chainsafe/backend.rs b/core/src/services/chainsafe/backend.rs index 88640c578d63..4983ef6d58f3 100644 --- a/core/src/services/chainsafe/backend.rs +++ b/core/src/services/chainsafe/backend.rs @@ -23,8 +23,6 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::parse_info; use super::core::ChainsafeCore; @@ -34,36 +32,9 @@ use super::lister::ChainsafeLister; use super::writer::ChainsafeWriter; use super::writer::ChainsafeWriters; use crate::raw::*; +use crate::services::ChainsafeConfig; use crate::*; -/// Config for Chainsafe services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct ChainsafeConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// api_key of this backend. - pub api_key: Option, - /// bucket_id of this backend. - /// - /// required. - pub bucket_id: String, -} - -impl Debug for ChainsafeConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("ChainsafeConfig"); - - d.field("root", &self.root) - .field("bucket_id", &self.bucket_id); - - d.finish_non_exhaustive() - } -} - impl Configurator for ChainsafeConfig { type Builder = ChainsafeBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/chainsafe/config.rs b/core/src/services/chainsafe/config.rs new file mode 100644 index 000000000000..73447ffe6b94 --- /dev/null +++ b/core/src/services/chainsafe/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Chainsafe services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct ChainsafeConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// api_key of this backend. + pub api_key: Option, + /// bucket_id of this backend. + /// + /// required. + pub bucket_id: String, +} + +impl Debug for ChainsafeConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("ChainsafeConfig"); + + d.field("root", &self.root) + .field("bucket_id", &self.bucket_id); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/chainsafe/mod.rs b/core/src/services/chainsafe/mod.rs index ed5335a738bf..86a7ff7bb554 100644 --- a/core/src/services/chainsafe/mod.rs +++ b/core/src/services/chainsafe/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::ChainsafeBuilder as Chainsafe; -pub use backend::ChainsafeConfig; - +#[cfg(feature = "services-chainsafe")] mod core; +#[cfg(feature = "services-chainsafe")] mod error; +#[cfg(feature = "services-chainsafe")] mod lister; +#[cfg(feature = "services-chainsafe")] mod writer; + +#[cfg(feature = "services-chainsafe")] +mod backend; +#[cfg(feature = "services-chainsafe")] +pub use backend::ChainsafeBuilder as Chainsafe; + +mod config; +pub use config::ChainsafeConfig; diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index f97bca593a5f..5442edb78e1d 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -23,44 +23,14 @@ use http::header; use http::Request; use http::StatusCode; use serde::Deserialize; -use serde::Serialize; use super::error::parse_error; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::CloudflareKvConfig; use crate::ErrorKind; use crate::*; -/// Cloudflare KV Service Support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct CloudflareKvConfig { - /// The token used to authenticate with CloudFlare. - pub token: Option, - /// The account ID used to authenticate with CloudFlare. Used as URI path parameter. - pub account_id: Option, - /// The namespace ID. Used as URI path parameter. - pub namespace_id: Option, - - /// Root within this backend. - pub root: Option, -} - -impl Debug for CloudflareKvConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("CloudflareKvConfig"); - - ds.field("root", &self.root); - ds.field("account_id", &self.account_id); - ds.field("namespace_id", &self.namespace_id); - - if self.token.is_some() { - ds.field("token", &""); - } - - ds.finish() - } -} - impl Configurator for CloudflareKvConfig { type Builder = CloudflareKvBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/cloudflare_kv/config.rs b/core/src/services/cloudflare_kv/config.rs new file mode 100644 index 000000000000..777b43c2a820 --- /dev/null +++ b/core/src/services/cloudflare_kv/config.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Cloudflare KV Service Support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct CloudflareKvConfig { + /// The token used to authenticate with CloudFlare. + pub token: Option, + /// The account ID used to authenticate with CloudFlare. Used as URI path parameter. + pub account_id: Option, + /// The namespace ID. Used as URI path parameter. + pub namespace_id: Option, + + /// Root within this backend. + pub root: Option, +} + +impl Debug for CloudflareKvConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("CloudflareKvConfig"); + + ds.field("root", &self.root); + ds.field("account_id", &self.account_id); + ds.field("namespace_id", &self.namespace_id); + + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/cloudflare_kv/mod.rs b/core/src/services/cloudflare_kv/mod.rs index 85011ee1e2e9..390bbd2912b5 100644 --- a/core/src/services/cloudflare_kv/mod.rs +++ b/core/src/services/cloudflare_kv/mod.rs @@ -15,8 +15,13 @@ // specific language governing permissions and limitations // under the License. -mod backend; +#[cfg(feature = "services-cloudflare-kv")] mod error; +#[cfg(feature = "services-cloudflare-kv")] +mod backend; +#[cfg(feature = "services-cloudflare-kv")] pub use backend::CloudflareKvBuilder as CloudflareKv; -pub use backend::CloudflareKvConfig; + +mod config; +pub use config::CloudflareKvConfig; diff --git a/core/src/services/compfs/backend.rs b/core/src/services/compfs/backend.rs index f866ccec076e..f24fd54e5d8c 100644 --- a/core/src/services/compfs/backend.rs +++ b/core/src/services/compfs/backend.rs @@ -20,25 +20,15 @@ use std::sync::Arc; use compio::dispatcher::Dispatcher; use compio::fs::OpenOptions; -use serde::Deserialize; -use serde::Serialize; use super::core::CompfsCore; use super::lister::CompfsLister; use super::reader::CompfsReader; use super::writer::CompfsWriter; use crate::raw::*; +use crate::services::CompfsConfig; use crate::*; -/// compio-based file system support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct CompfsConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, -} - impl Configurator for CompfsConfig { type Builder = CompfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/compfs/config.rs b/core/src/services/compfs/config.rs new file mode 100644 index 000000000000..6dd17193210a --- /dev/null +++ b/core/src/services/compfs/config.rs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// compio-based file system support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct CompfsConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, +} diff --git a/core/src/services/compfs/mod.rs b/core/src/services/compfs/mod.rs index 0fd86a7e3cfc..fe99a6d00562 100644 --- a/core/src/services/compfs/mod.rs +++ b/core/src/services/compfs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -pub use backend::CompfsBuilder as Compfs; -pub use backend::CompfsConfig; - -mod backend; +#[cfg(feature = "services-compfs")] mod core; +#[cfg(feature = "services-compfs")] mod lister; +#[cfg(feature = "services-compfs")] mod reader; +#[cfg(feature = "services-compfs")] mod writer; + +#[cfg(feature = "services-compfs")] +mod backend; +#[cfg(feature = "services-compfs")] +pub use backend::CompfsBuilder as Compfs; + +mod config; +pub use config::CompfsConfig; diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 478aecb38608..a3768395c665 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -25,47 +25,16 @@ use log::debug; use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; use super::lister::CosLister; use super::writer::CosWriter; +use super::writer::CosWriters; use crate::raw::*; -use crate::services::cos::writer::CosWriters; +use crate::services::CosConfig; use crate::*; -/// Tencent-Cloud COS services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -pub struct CosConfig { - /// Root of this backend. - pub root: Option, - /// Endpoint of this backend. - pub endpoint: Option, - /// Secret ID of this backend. - pub secret_id: Option, - /// Secret key of this backend. - pub secret_key: Option, - /// Bucket of this backend. - pub bucket: Option, - /// Disable config load so that opendal will not load config from - pub disable_config_load: bool, -} - -impl Debug for CosConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CosConfig") - .field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("secret_id", &"") - .field("secret_key", &"") - .field("bucket", &self.bucket) - .finish_non_exhaustive() - } -} - impl Configurator for CosConfig { type Builder = CosBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/cos/config.rs b/core/src/services/cos/config.rs new file mode 100644 index 000000000000..166b8159a183 --- /dev/null +++ b/core/src/services/cos/config.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Tencent-Cloud COS services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +pub struct CosConfig { + /// Root of this backend. + pub root: Option, + /// Endpoint of this backend. + pub endpoint: Option, + /// Secret ID of this backend. + pub secret_id: Option, + /// Secret key of this backend. + pub secret_key: Option, + /// Bucket of this backend. + pub bucket: Option, + /// Disable config load so that opendal will not load config from + pub disable_config_load: bool, +} + +impl Debug for CosConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CosConfig") + .field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("secret_id", &"") + .field("secret_key", &"") + .field("bucket", &self.bucket) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/cos/mod.rs b/core/src/services/cos/mod.rs index e035b71abccc..9713dc39dc5b 100644 --- a/core/src/services/cos/mod.rs +++ b/core/src/services/cos/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::CosBuilder as Cos; -pub use backend::CosConfig; - +#[cfg(feature = "services-cos")] mod core; +#[cfg(feature = "services-cos")] mod error; +#[cfg(feature = "services-cos")] mod lister; +#[cfg(feature = "services-cos")] mod writer; + +#[cfg(feature = "services-cos")] +mod backend; +#[cfg(feature = "services-cos")] +pub use backend::CosBuilder as Cos; + +mod config; +pub use config::CosConfig; diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs index c30b622a3a06..3e37a532adf8 100644 --- a/core/src/services/d1/backend.rs +++ b/core/src/services/d1/backend.rs @@ -22,50 +22,16 @@ use bytes::Buf; use http::header; use http::Request; use http::StatusCode; -use serde::Deserialize; -use serde::Serialize; use serde_json::Value; use super::error::parse_error; use super::model::D1Response; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::D1Config; use crate::ErrorKind; use crate::*; -/// Config for [Cloudflare D1](https://developers.cloudflare.com/d1) backend support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct D1Config { - /// Set the token of cloudflare api. - pub token: Option, - /// Set the account id of cloudflare api. - pub account_id: Option, - /// Set the database id of cloudflare api. - pub database_id: Option, - - /// Set the working directory of OpenDAL. - pub root: Option, - /// Set the table of D1 Database. - pub table: Option, - /// Set the key field of D1 Database. - pub key_field: Option, - /// Set the value field of D1 Database. - pub value_field: Option, -} - -impl Debug for D1Config { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("D1Config"); - ds.field("root", &self.root); - ds.field("table", &self.table); - ds.field("key_field", &self.key_field); - ds.field("value_field", &self.value_field); - ds.finish_non_exhaustive() - } -} - impl Configurator for D1Config { type Builder = D1Builder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/d1/config.rs b/core/src/services/d1/config.rs new file mode 100644 index 000000000000..0269f4124f2b --- /dev/null +++ b/core/src/services/d1/config.rs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for [Cloudflare D1](https://developers.cloudflare.com/d1) backend support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct D1Config { + /// Set the token of cloudflare api. + pub token: Option, + /// Set the account id of cloudflare api. + pub account_id: Option, + /// Set the database id of cloudflare api. + pub database_id: Option, + + /// Set the working directory of OpenDAL. + pub root: Option, + /// Set the table of D1 Database. + pub table: Option, + /// Set the key field of D1 Database. + pub key_field: Option, + /// Set the value field of D1 Database. + pub value_field: Option, +} + +impl Debug for D1Config { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("D1Config"); + ds.field("root", &self.root); + ds.field("table", &self.table); + ds.field("key_field", &self.key_field); + ds.field("value_field", &self.value_field); + ds.finish_non_exhaustive() + } +} diff --git a/core/src/services/d1/mod.rs b/core/src/services/d1/mod.rs index 1fe63408affe..6f13716f574e 100644 --- a/core/src/services/d1/mod.rs +++ b/core/src/services/d1/mod.rs @@ -15,9 +15,15 @@ // specific language governing permissions and limitations // under the License. -mod backend; +#[cfg(feature = "services-d1")] mod error; +#[cfg(feature = "services-d1")] mod model; +#[cfg(feature = "services-d1")] +mod backend; +#[cfg(feature = "services-d1")] pub use backend::D1Builder as D1; -pub use backend::D1Config; + +mod config; +pub use config::D1Config; diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index 9b8bdf55e647..49312c673908 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -19,20 +19,12 @@ use std::fmt::Debug; use std::fmt::Formatter; use dashmap::DashMap; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::typed_kv; use crate::raw::Access; +use crate::services::DashmapConfig; use crate::*; -/// [dashmap](https://github.com/xacrimon/dashmap) backend support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct DashmapConfig { - /// The root path for dashmap. - pub root: Option, -} - impl Configurator for DashmapConfig { type Builder = DashmapBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/dashmap/config.rs b/core/src/services/dashmap/config.rs new file mode 100644 index 000000000000..f80fa15c9dff --- /dev/null +++ b/core/src/services/dashmap/config.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// [dashmap](https://github.com/xacrimon/dashmap) backend support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct DashmapConfig { + /// The root path for dashmap. + pub root: Option, +} diff --git a/core/src/services/dashmap/mod.rs b/core/src/services/dashmap/mod.rs index 0cd7e387fa24..3b11d9b665f3 100644 --- a/core/src/services/dashmap/mod.rs +++ b/core/src/services/dashmap/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-dashmap")] mod backend; +#[cfg(feature = "services-dashmap")] pub use backend::DashmapBuilder as Dashmap; -pub use backend::DashmapConfig; + +mod config; +pub use config::DashmapConfig; diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index 1f5b16ff37af..bf1152daea09 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -23,41 +23,15 @@ use bytes::Buf; use http::StatusCode; use log::debug; use serde::Deserialize; -use serde::Serialize; use super::core::DbfsCore; use super::error::parse_error; use super::lister::DbfsLister; use super::writer::DbfsWriter; use crate::raw::*; +use crate::services::DbfsConfig; use crate::*; -/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct DbfsConfig { - /// The root for dbfs. - pub root: Option, - /// The endpoint for dbfs. - pub endpoint: Option, - /// The token for dbfs. - pub token: Option, -} - -impl Debug for DbfsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("DbfsConfig"); - - ds.field("root", &self.root); - ds.field("endpoint", &self.endpoint); - - if self.token.is_some() { - ds.field("token", &""); - } - - ds.finish() - } -} - impl Configurator for DbfsConfig { type Builder = DbfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/dbfs/config.rs b/core/src/services/dbfs/config.rs new file mode 100644 index 000000000000..285ddf71ba81 --- /dev/null +++ b/core/src/services/dbfs/config.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct DbfsConfig { + /// The root for dbfs. + pub root: Option, + /// The endpoint for dbfs. + pub endpoint: Option, + /// The token for dbfs. + pub token: Option, +} + +impl Debug for DbfsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("DbfsConfig"); + + ds.field("root", &self.root); + ds.field("endpoint", &self.endpoint); + + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/dbfs/mod.rs b/core/src/services/dbfs/mod.rs index 98ac4f40d306..de4774f7d1a5 100644 --- a/core/src/services/dbfs/mod.rs +++ b/core/src/services/dbfs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -pub use backend::DbfsBuilder as Dbfs; -pub use backend::DbfsConfig; - -mod backend; +#[cfg(feature = "services-dbfs")] mod core; +#[cfg(feature = "services-dbfs")] mod error; +#[cfg(feature = "services-dbfs")] mod lister; +#[cfg(feature = "services-dbfs")] mod writer; + +#[cfg(feature = "services-dbfs")] +mod backend; +#[cfg(feature = "services-dbfs")] +pub use backend::DbfsBuilder as Dbfs; + +mod config; +pub use config::DbfsConfig; diff --git a/core/src/services/dropbox/builder.rs b/core/src/services/dropbox/builder.rs index 5b73de42003b..51d852e1e113 100644 --- a/core/src/services/dropbox/builder.rs +++ b/core/src/services/dropbox/builder.rs @@ -21,41 +21,15 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::Mutex; use super::backend::DropboxBackend; use super::core::DropboxCore; use super::core::DropboxSigner; use crate::raw::*; +use crate::services::DropboxConfig; use crate::*; -/// Config for [Dropbox](https://www.dropbox.com/) backend support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct DropboxConfig { - /// root path for dropbox. - pub root: Option, - /// access token for dropbox. - pub access_token: Option, - /// refresh_token for dropbox. - pub refresh_token: Option, - /// client_id for dropbox. - pub client_id: Option, - /// client_secret for dropbox. - pub client_secret: Option, -} - -impl Debug for DropboxConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DropBoxConfig") - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for DropboxConfig { type Builder = DropboxBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/dropbox/config.rs b/core/src/services/dropbox/config.rs new file mode 100644 index 000000000000..e44222da36f9 --- /dev/null +++ b/core/src/services/dropbox/config.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for [Dropbox](https://www.dropbox.com/) backend support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct DropboxConfig { + /// root path for dropbox. + pub root: Option, + /// access token for dropbox. + pub access_token: Option, + /// refresh_token for dropbox. + pub refresh_token: Option, + /// client_id for dropbox. + pub client_id: Option, + /// client_secret for dropbox. + pub client_secret: Option, +} + +impl Debug for DropboxConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DropBoxConfig") + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/dropbox/mod.rs b/core/src/services/dropbox/mod.rs index 3aaa93db9add..21397e855408 100644 --- a/core/src/services/dropbox/mod.rs +++ b/core/src/services/dropbox/mod.rs @@ -15,12 +15,21 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-dropbox")] mod backend; -mod builder; +#[cfg(feature = "services-dropbox")] mod core; +#[cfg(feature = "services-dropbox")] mod error; +#[cfg(feature = "services-dropbox")] mod lister; +#[cfg(feature = "services-dropbox")] mod writer; +#[cfg(feature = "services-dropbox")] +mod builder; +#[cfg(feature = "services-dropbox")] pub use builder::DropboxBuilder as Dropbox; -pub use builder::DropboxConfig; + +mod config; +pub use config::DropboxConfig; diff --git a/core/src/services/etcd/backend.rs b/core/src/services/etcd/backend.rs index fa7fb348207c..d4e3b721bd17 100644 --- a/core/src/services/etcd/backend.rs +++ b/core/src/services/etcd/backend.rs @@ -27,80 +27,15 @@ use etcd_client::Error as EtcdError; use etcd_client::GetOptions; use etcd_client::Identity; use etcd_client::TlsOptions; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::EtcdConfig; use crate::*; const DEFAULT_ETCD_ENDPOINTS: &str = "http://127.0.0.1:2379"; -/// Config for Etcd services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct EtcdConfig { - /// network address of the Etcd services. - /// If use https, must set TLS options: `ca_path`, `cert_path`, `key_path`. - /// e.g. "127.0.0.1:23790,127.0.0.1:23791,127.0.0.1:23792" or "http://127.0.0.1:23790,http://127.0.0.1:23791,http://127.0.0.1:23792" or "https://127.0.0.1:23790,https://127.0.0.1:23791,https://127.0.0.1:23792" - /// - /// default is "http://127.0.0.1:2379" - pub endpoints: Option, - /// the username to connect etcd service. - /// - /// default is None - pub username: Option, - /// the password for authentication - /// - /// default is None - pub password: Option, - /// the working directory of the etcd service. Can be "/path/to/dir" - /// - /// default is "/" - pub root: Option, - /// certificate authority file path - /// - /// default is None - pub ca_path: Option, - /// cert path - /// - /// default is None - pub cert_path: Option, - /// key path - /// - /// default is None - pub key_path: Option, -} - -impl Debug for EtcdConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("EtcdConfig"); - - ds.field("root", &self.root); - if let Some(endpoints) = self.endpoints.clone() { - ds.field("endpoints", &endpoints); - } - if let Some(username) = self.username.clone() { - ds.field("username", &username); - } - if self.password.is_some() { - ds.field("password", &""); - } - if let Some(ca_path) = self.ca_path.clone() { - ds.field("ca_path", &ca_path); - } - if let Some(cert_path) = self.cert_path.clone() { - ds.field("cert_path", &cert_path); - } - if let Some(key_path) = self.key_path.clone() { - ds.field("key_path", &key_path); - } - ds.finish() - } -} - impl Configurator for EtcdConfig { type Builder = EtcdBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/etcd/config.rs b/core/src/services/etcd/config.rs new file mode 100644 index 000000000000..b5ee4e4f42ad --- /dev/null +++ b/core/src/services/etcd/config.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Etcd services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct EtcdConfig { + /// network address of the Etcd services. + /// If use https, must set TLS options: `ca_path`, `cert_path`, `key_path`. + /// e.g. "127.0.0.1:23790,127.0.0.1:23791,127.0.0.1:23792" or "http://127.0.0.1:23790,http://127.0.0.1:23791,http://127.0.0.1:23792" or "https://127.0.0.1:23790,https://127.0.0.1:23791,https://127.0.0.1:23792" + /// + /// default is "http://127.0.0.1:2379" + pub endpoints: Option, + /// the username to connect etcd service. + /// + /// default is None + pub username: Option, + /// the password for authentication + /// + /// default is None + pub password: Option, + /// the working directory of the etcd service. Can be "/path/to/dir" + /// + /// default is "/" + pub root: Option, + /// certificate authority file path + /// + /// default is None + pub ca_path: Option, + /// cert path + /// + /// default is None + pub cert_path: Option, + /// key path + /// + /// default is None + pub key_path: Option, +} + +impl Debug for EtcdConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("EtcdConfig"); + + ds.field("root", &self.root); + if let Some(endpoints) = self.endpoints.clone() { + ds.field("endpoints", &endpoints); + } + if let Some(username) = self.username.clone() { + ds.field("username", &username); + } + if self.password.is_some() { + ds.field("password", &""); + } + if let Some(ca_path) = self.ca_path.clone() { + ds.field("ca_path", &ca_path); + } + if let Some(cert_path) = self.cert_path.clone() { + ds.field("cert_path", &cert_path); + } + if let Some(key_path) = self.key_path.clone() { + ds.field("key_path", &key_path); + } + ds.finish() + } +} diff --git a/core/src/services/etcd/mod.rs b/core/src/services/etcd/mod.rs index ddb370da76af..f55659e3e4c9 100644 --- a/core/src/services/etcd/mod.rs +++ b/core/src/services/etcd/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-etcd")] mod backend; +#[cfg(feature = "services-etcd")] pub use backend::EtcdBuilder as Etcd; -pub use backend::EtcdConfig; + +mod config; +pub use config::EtcdConfig; diff --git a/core/src/services/foundationdb/backend.rs b/core/src/services/foundationdb/backend.rs index 553b6a06087d..4d4adfa52fd2 100644 --- a/core/src/services/foundationdb/backend.rs +++ b/core/src/services/foundationdb/backend.rs @@ -21,40 +21,16 @@ use std::sync::Arc; use foundationdb::api::NetworkAutoStop; use foundationdb::Database; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::FoundationdbConfig; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; -/// [foundationdb](https://www.foundationdb.org/) service support. -///Config for FoundationDB. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct FoundationdbConfig { - ///root of the backend. - pub root: Option, - ///config_path for the backend. - pub config_path: Option, -} - -impl Debug for FoundationdbConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("FoundationConfig"); - - ds.field("root", &self.root); - ds.field("config_path", &self.config_path); - - ds.finish() - } -} - impl Configurator for FoundationdbConfig { type Builder = FoundationdbBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/foundationdb/config.rs b/core/src/services/foundationdb/config.rs new file mode 100644 index 000000000000..b210bf8da4d7 --- /dev/null +++ b/core/src/services/foundationdb/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// [foundationdb](https://www.foundationdb.org/) service support. +///Config for FoundationDB. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct FoundationdbConfig { + ///root of the backend. + pub root: Option, + ///config_path for the backend. + pub config_path: Option, +} + +impl Debug for FoundationdbConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("FoundationConfig"); + + ds.field("root", &self.root); + ds.field("config_path", &self.config_path); + + ds.finish() + } +} diff --git a/core/src/services/foundationdb/mod.rs b/core/src/services/foundationdb/mod.rs index a761e25a47ed..dd8650ff7132 100644 --- a/core/src/services/foundationdb/mod.rs +++ b/core/src/services/foundationdb/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-foundationdb")] mod backend; - +#[cfg(feature = "services-foundationdb")] pub use backend::FoundationdbBuilder as Foundationdb; -pub use backend::FoundationdbConfig; + +mod config; +pub use config::FoundationdbConfig; diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index af1f163187c8..26fa8dd6c92b 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -22,8 +22,6 @@ use std::sync::Arc; use chrono::DateTime; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::lister::FsLister; @@ -31,20 +29,9 @@ use super::reader::FsReader; use super::writer::FsWriter; use super::writer::FsWriters; use crate::raw::*; +use crate::services::FsConfig; use crate::*; -/// config for file system -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct FsConfig { - /// root dir for backend - pub root: Option, - - /// tmp dir for atomic write - pub atomic_write_dir: Option, -} - impl Configurator for FsConfig { type Builder = FsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/fs/config.rs b/core/src/services/fs/config.rs new file mode 100644 index 000000000000..e79b53641422 --- /dev/null +++ b/core/src/services/fs/config.rs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// config for file system +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct FsConfig { + /// root dir for backend + pub root: Option, + + /// tmp dir for atomic write + pub atomic_write_dir: Option, +} diff --git a/core/src/services/fs/mod.rs b/core/src/services/fs/mod.rs index 6ffe4896a864..bf914a75eaf7 100644 --- a/core/src/services/fs/mod.rs +++ b/core/src/services/fs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::FsBuilder as Fs; -pub use backend::FsConfig; - +#[cfg(feature = "services-fs")] mod core; +#[cfg(feature = "services-fs")] mod lister; +#[cfg(feature = "services-fs")] mod reader; +#[cfg(feature = "services-fs")] mod writer; + +#[cfg(feature = "services-fs")] +mod backend; +#[cfg(feature = "services-fs")] +pub use backend::FsBuilder as Fs; + +mod config; +pub use config::FsConfig; diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 1d5cb84b0357..144ba9aff7ad 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -26,8 +26,6 @@ use bb8::PooledConnection; use bb8::RunError; use http::Uri; use log::debug; -use serde::Deserialize; -use serde::Serialize; use suppaftp::list::File; use suppaftp::types::FileType; use suppaftp::types::Response; @@ -44,32 +42,9 @@ use super::lister::FtpLister; use super::reader::FtpReader; use super::writer::FtpWriter; use crate::raw::*; +use crate::services::FtpConfig; use crate::*; -/// Config for Ftp services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct FtpConfig { - /// endpoint of this backend - pub endpoint: Option, - /// root of this backend - pub root: Option, - /// user of this backend - pub user: Option, - /// password of this backend - pub password: Option, -} - -impl Debug for FtpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FtpConfig") - .field("endpoint", &self.endpoint) - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for FtpConfig { type Builder = FtpBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/ftp/config.rs b/core/src/services/ftp/config.rs new file mode 100644 index 000000000000..1f1a1c32b454 --- /dev/null +++ b/core/src/services/ftp/config.rs @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Ftp services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct FtpConfig { + /// endpoint of this backend + pub endpoint: Option, + /// root of this backend + pub root: Option, + /// user of this backend + pub user: Option, + /// password of this backend + pub password: Option, +} + +impl Debug for FtpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FtpConfig") + .field("endpoint", &self.endpoint) + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/ftp/mod.rs b/core/src/services/ftp/mod.rs index 35a0ba5734ac..e1b7440fac72 100644 --- a/core/src/services/ftp/mod.rs +++ b/core/src/services/ftp/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::FtpBuilder as Ftp; -pub use backend::FtpConfig; - +#[cfg(feature = "services-ftp")] mod err; +#[cfg(feature = "services-ftp")] mod lister; +#[cfg(feature = "services-ftp")] mod reader; +#[cfg(feature = "services-ftp")] mod writer; + +#[cfg(feature = "services-ftp")] +mod backend; +#[cfg(feature = "services-ftp")] +pub use backend::FtpBuilder as Ftp; + +mod config; +pub use config::FtpConfig; diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index df3fab369f5a..dd26a67ab3f2 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -28,7 +28,6 @@ use reqsign::GoogleSigner; use reqsign::GoogleTokenLoad; use reqsign::GoogleTokenLoader; use serde::Deserialize; -use serde::Serialize; use serde_json; use super::core::*; @@ -37,60 +36,12 @@ use super::lister::GcsLister; use super::writer::GcsWriter; use super::writer::GcsWriters; use crate::raw::*; +use crate::services::GcsConfig; use crate::*; const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com"; const DEFAULT_GCS_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_write"; -/// [Google Cloud Storage](https://cloud.google.com/storage) services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct GcsConfig { - /// root URI, all operations happens under `root` - pub root: Option, - /// bucket name - pub bucket: String, - /// endpoint URI of GCS service, - /// default is `https://storage.googleapis.com` - pub endpoint: Option, - /// Scope for gcs. - pub scope: Option, - /// Service Account for gcs. - pub service_account: Option, - /// Credentials string for GCS service OAuth2 authentication. - pub credential: Option, - /// Local path to credentials file for GCS service OAuth2 authentication. - pub credential_path: Option, - /// The predefined acl for GCS. - pub predefined_acl: Option, - /// The default storage class used by gcs. - pub default_storage_class: Option, - /// Allow opendal to send requests without signing when credentials are not - /// loaded. - pub allow_anonymous: bool, - /// Disable attempting to load credentials from the GCE metadata server when - /// running within Google Cloud. - pub disable_vm_metadata: bool, - /// Disable loading configuration from the environment. - pub disable_config_load: bool, - /// A Google Cloud OAuth2 token. - /// - /// Takes precedence over `credential` and `credential_path`. - pub token: Option, -} - -impl Debug for GcsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("GcsConfig") - .field("root", &self.root) - .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint) - .field("scope", &self.scope) - .finish_non_exhaustive() - } -} - impl Configurator for GcsConfig { type Builder = GcsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/gcs/config.rs b/core/src/services/gcs/config.rs new file mode 100644 index 000000000000..31e60d0ba84d --- /dev/null +++ b/core/src/services/gcs/config.rs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// [Google Cloud Storage](https://cloud.google.com/storage) services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GcsConfig { + /// root URI, all operations happens under `root` + pub root: Option, + /// bucket name + pub bucket: String, + /// endpoint URI of GCS service, + /// default is `https://storage.googleapis.com` + pub endpoint: Option, + /// Scope for gcs. + pub scope: Option, + /// Service Account for gcs. + pub service_account: Option, + /// Credentials string for GCS service OAuth2 authentication. + pub credential: Option, + /// Local path to credentials file for GCS service OAuth2 authentication. + pub credential_path: Option, + /// The predefined acl for GCS. + pub predefined_acl: Option, + /// The default storage class used by gcs. + pub default_storage_class: Option, + /// Allow opendal to send requests without signing when credentials are not + /// loaded. + pub allow_anonymous: bool, + /// Disable attempting to load credentials from the GCE metadata server when + /// running within Google Cloud. + pub disable_vm_metadata: bool, + /// Disable loading configuration from the environment. + pub disable_config_load: bool, + /// A Google Cloud OAuth2 token. + /// + /// Takes precedence over `credential` and `credential_path`. + pub token: Option, +} + +impl Debug for GcsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GcsConfig") + .field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("scope", &self.scope) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/gcs/mod.rs b/core/src/services/gcs/mod.rs index 1dcaa9ca0632..2f260dd081bb 100644 --- a/core/src/services/gcs/mod.rs +++ b/core/src/services/gcs/mod.rs @@ -15,12 +15,21 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::GcsBuilder as Gcs; -pub use backend::GcsConfig; - +#[cfg(feature = "services-gcs")] mod core; +#[cfg(feature = "services-gcs")] mod error; +#[cfg(feature = "services-gcs")] mod lister; +#[cfg(feature = "services-gcs")] mod uri; +#[cfg(feature = "services-gcs")] mod writer; + +#[cfg(feature = "services-gcs")] +mod backend; +#[cfg(feature = "services-gcs")] +pub use backend::GcsBuilder as Gcs; + +mod config; +pub use config::GcsConfig; diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs index 3628488b819e..5c6c18712345 100644 --- a/core/src/services/gdrive/builder.rs +++ b/core/src/services/gdrive/builder.rs @@ -22,8 +22,6 @@ use std::sync::Arc; use chrono::DateTime; use chrono::Utc; use log::debug; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::Mutex; use super::backend::GdriveBackend; @@ -34,34 +32,10 @@ use crate::raw::PathCacher; use crate::services::gdrive::core::GdriveCore; use crate::services::gdrive::core::GdrivePathQuery; use crate::services::gdrive::core::GdriveSigner; +use crate::services::GdriveConfig; use crate::Scheme; use crate::*; -/// [GoogleDrive](https://drive.google.com/) configuration. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct GdriveConfig { - /// The root for gdrive - pub root: Option, - /// Access token for gdrive. - pub access_token: Option, - /// Refresh token for gdrive. - pub refresh_token: Option, - /// Client id for gdrive. - pub client_id: Option, - /// Client secret for gdrive. - pub client_secret: Option, -} - -impl Debug for GdriveConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("GdriveConfig") - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for GdriveConfig { type Builder = GdriveBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/gdrive/config.rs b/core/src/services/gdrive/config.rs new file mode 100644 index 000000000000..85df5f932a8a --- /dev/null +++ b/core/src/services/gdrive/config.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// [GoogleDrive](https://drive.google.com/) configuration. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GdriveConfig { + /// The root for gdrive + pub root: Option, + /// Access token for gdrive. + pub access_token: Option, + /// Refresh token for gdrive. + pub refresh_token: Option, + /// Client id for gdrive. + pub client_id: Option, + /// Client secret for gdrive. + pub client_secret: Option, +} + +impl Debug for GdriveConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GdriveConfig") + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs index 7b37822ee0e5..a98995cb6b88 100644 --- a/core/src/services/gdrive/mod.rs +++ b/core/src/services/gdrive/mod.rs @@ -15,12 +15,21 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-gdrive")] mod backend; -mod builder; +#[cfg(feature = "services-gdrive")] mod core; +#[cfg(feature = "services-gdrive")] mod error; - -pub use builder::GdriveBuilder as Gdrive; -pub use builder::GdriveConfig; +#[cfg(feature = "services-gdrive")] mod lister; +#[cfg(feature = "services-gdrive")] mod writer; + +#[cfg(feature = "services-gdrive")] +mod builder; +#[cfg(feature = "services-gdrive")] +pub use builder::GdriveBuilder as Gdrive; + +mod config; +pub use config::GdriveConfig; diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs index 5a6fc305451a..3d8a5cbce766 100644 --- a/core/src/services/ghac/backend.rs +++ b/core/src/services/ghac/backend.rs @@ -37,6 +37,7 @@ use serde::Serialize; use super::error::parse_error; use super::writer::GhacWriter; use crate::raw::*; +use crate::services::GhacConfig; use crate::*; /// The base url for cache url. @@ -81,21 +82,6 @@ fn value_or_env( }) } -/// Config for GitHub Action Cache Services support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct GhacConfig { - /// The root path for ghac. - pub root: Option, - /// The version that used by cache. - pub version: Option, - /// The endpoint for ghac service. - pub endpoint: Option, - /// The runtime token for ghac service. - pub runtime_token: Option, -} - impl Configurator for GhacConfig { type Builder = GhacBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/ghac/config.rs b/core/src/services/ghac/config.rs new file mode 100644 index 000000000000..7958c06302ed --- /dev/null +++ b/core/src/services/ghac/config.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for GitHub Action Cache Services support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GhacConfig { + /// The root path for ghac. + pub root: Option, + /// The version that used by cache. + pub version: Option, + /// The endpoint for ghac service. + pub endpoint: Option, + /// The runtime token for ghac service. + pub runtime_token: Option, +} diff --git a/core/src/services/ghac/mod.rs b/core/src/services/ghac/mod.rs index 2a87b2258399..fabc5bcc15a3 100644 --- a/core/src/services/ghac/mod.rs +++ b/core/src/services/ghac/mod.rs @@ -15,9 +15,15 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-ghac")] +mod error; +#[cfg(feature = "services-ghac")] +mod writer; + +#[cfg(feature = "services-ghac")] mod backend; +#[cfg(feature = "services-ghac")] pub use backend::GhacBuilder as Ghac; -pub use backend::GhacConfig; -mod error; -mod writer; +mod config; +pub use config::GhacConfig; diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs index f33bd0554a2b..d51bff524491 100644 --- a/core/src/services/github/backend.rs +++ b/core/src/services/github/backend.rs @@ -23,8 +23,6 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::Entry; use super::core::GithubCore; @@ -33,45 +31,9 @@ use super::lister::GithubLister; use super::writer::GithubWriter; use super::writer::GithubWriters; use crate::raw::*; +use crate::services::GithubConfig; use crate::*; -/// Config for GitHub services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct GithubConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// GitHub access_token. - /// - /// optional. - /// If not provided, the backend will only support read operations for public repositories. - /// And rate limit will be limited to 60 requests per hour. - pub token: Option, - /// GitHub repo owner. - /// - /// required. - pub owner: String, - /// GitHub repo name. - /// - /// required. - pub repo: String, -} - -impl Debug for GithubConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("GithubConfig"); - - d.field("root", &self.root) - .field("owner", &self.owner) - .field("repo", &self.repo); - - d.finish_non_exhaustive() - } -} - impl Configurator for GithubConfig { type Builder = GithubBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/github/config.rs b/core/src/services/github/config.rs new file mode 100644 index 000000000000..c3927f89ffab --- /dev/null +++ b/core/src/services/github/config.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for GitHub services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GithubConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// GitHub access_token. + /// + /// optional. + /// If not provided, the backend will only support read operations for public repositories. + /// And rate limit will be limited to 60 requests per hour. + pub token: Option, + /// GitHub repo owner. + /// + /// required. + pub owner: String, + /// GitHub repo name. + /// + /// required. + pub repo: String, +} + +impl Debug for GithubConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("GithubConfig"); + + d.field("root", &self.root) + .field("owner", &self.owner) + .field("repo", &self.repo); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/github/mod.rs b/core/src/services/github/mod.rs index 06418c19347b..bf1b1c5a4b9b 100644 --- a/core/src/services/github/mod.rs +++ b/core/src/services/github/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::GithubBuilder as Github; -pub use backend::GithubConfig; - +#[cfg(feature = "services-github")] mod core; +#[cfg(feature = "services-github")] mod error; +#[cfg(feature = "services-github")] mod lister; +#[cfg(feature = "services-github")] mod writer; + +#[cfg(feature = "services-github")] +mod backend; +#[cfg(feature = "services-github")] +pub use backend::GithubBuilder as Github; + +mod config; +pub use config::GithubConfig; diff --git a/core/src/services/gridfs/backend.rs b/core/src/services/gridfs/backend.rs index 7abfa0d79764..5b1966f029e1 100644 --- a/core/src/services/gridfs/backend.rs +++ b/core/src/services/gridfs/backend.rs @@ -24,57 +24,28 @@ use mongodb::bson::doc; use mongodb::gridfs::GridFsBucket; use mongodb::options::ClientOptions; use mongodb::options::GridFsBucketOptions; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use crate::raw::adapters::kv; -use crate::raw::new_std_io_error; use crate::raw::*; use crate::*; -/// Config for Grid file system support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct GridFsConfig { - /// The connection string of the MongoDB service. - pub connection_string: Option, - /// The database name of the MongoDB GridFs service to read/write. - pub database: Option, - /// The bucket name of the MongoDB GridFs service to read/write. - pub bucket: Option, - /// The chunk size of the MongoDB GridFs service used to break the user file into chunks. - pub chunk_size: Option, - /// The working directory, all operations will be performed under it. - pub root: Option, -} - -impl Debug for GridFsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("GridFsConfig") - .field("database", &self.database) - .field("bucket", &self.bucket) - .field("chunk_size", &self.chunk_size) - .field("root", &self.root) - .finish() - } -} +use super::config::GridfsConfig; -impl Configurator for GridFsConfig { - type Builder = GridFsBuilder; +impl Configurator for GridfsConfig { + type Builder = GridfsBuilder; fn into_builder(self) -> Self::Builder { - GridFsBuilder { config: self } + GridfsBuilder { config: self } } } #[doc = include_str!("docs.md")] #[derive(Default)] -pub struct GridFsBuilder { - config: GridFsConfig, +pub struct GridfsBuilder { + config: GridfsConfig, } -impl Debug for GridFsBuilder { +impl Debug for GridfsBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let mut d = f.debug_struct("GridFsBuilder"); d.field("config", &self.config); @@ -82,7 +53,7 @@ impl Debug for GridFsBuilder { } } -impl GridFsBuilder { +impl GridfsBuilder { /// Set the connection_string of the MongoDB service. /// /// This connection string is used to connect to the MongoDB service. It typically follows the format: @@ -152,9 +123,9 @@ impl GridFsBuilder { } } -impl Builder for GridFsBuilder { - const SCHEME: Scheme = Scheme::Mongodb; - type Config = GridFsConfig; +impl Builder for GridfsBuilder { + const SCHEME: Scheme = Scheme::Gridfs; + type Config = GridfsConfig; fn build(self) -> Result { let conn = match &self.config.connection_string.clone() { diff --git a/core/src/services/gridfs/config.rs b/core/src/services/gridfs/config.rs new file mode 100644 index 000000000000..e7d148d2694e --- /dev/null +++ b/core/src/services/gridfs/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Grid file system support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct GridfsConfig { + /// The connection string of the MongoDB service. + pub connection_string: Option, + /// The database name of the MongoDB GridFs service to read/write. + pub database: Option, + /// The bucket name of the MongoDB GridFs service to read/write. + pub bucket: Option, + /// The chunk size of the MongoDB GridFs service used to break the user file into chunks. + pub chunk_size: Option, + /// The working directory, all operations will be performed under it. + pub root: Option, +} + +impl Debug for GridfsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GridFsConfig") + .field("database", &self.database) + .field("bucket", &self.bucket) + .field("chunk_size", &self.chunk_size) + .field("root", &self.root) + .finish() + } +} diff --git a/core/src/services/gridfs/mod.rs b/core/src/services/gridfs/mod.rs index 676a4d128d10..db2a2a8aecf5 100644 --- a/core/src/services/gridfs/mod.rs +++ b/core/src/services/gridfs/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-gridfs")] mod backend; -pub use backend::GridFsBuilder as Gridfs; -pub use backend::GridFsConfig; +#[cfg(feature = "services-gridfs")] +pub use backend::GridfsBuilder as Gridfs; + +mod config; +pub use config::GridfsConfig; diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index ff240c248c28..e4aba77ec48a 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -24,53 +24,15 @@ use std::sync::Arc; use futures::AsyncWriteExt; use log::debug; -use serde::Deserialize; -use serde::Serialize; use uuid::Uuid; use super::lister::HdfsLister; use super::writer::HdfsWriter; use crate::raw::*; use crate::services::hdfs::reader::HdfsReader; +use crate::services::HdfsConfig; use crate::*; -/// [Hadoop Distributed File System (HDFSâ„¢)](https://hadoop.apache.org/) support. -/// -/// Config for Hdfs services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct HdfsConfig { - /// work dir of this backend - pub root: Option, - /// name node of this backend - pub name_node: Option, - /// kerberos_ticket_cache_path of this backend - pub kerberos_ticket_cache_path: Option, - /// user of this backend - pub user: Option, - /// enable the append capacity - pub enable_append: bool, - /// atomic_write_dir of this backend - pub atomic_write_dir: Option, -} - -impl Debug for HdfsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("HdfsConfig") - .field("root", &self.root) - .field("name_node", &self.name_node) - .field( - "kerberos_ticket_cache_path", - &self.kerberos_ticket_cache_path, - ) - .field("user", &self.user) - .field("enable_append", &self.enable_append) - .field("atomic_write_dir", &self.atomic_write_dir) - .finish_non_exhaustive() - } -} - impl Configurator for HdfsConfig { type Builder = HdfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/hdfs/config.rs b/core/src/services/hdfs/config.rs new file mode 100644 index 000000000000..f4fbb129e839 --- /dev/null +++ b/core/src/services/hdfs/config.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// [Hadoop Distributed File System (HDFSâ„¢)](https://hadoop.apache.org/) support. +/// +/// Config for Hdfs services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct HdfsConfig { + /// work dir of this backend + pub root: Option, + /// name node of this backend + pub name_node: Option, + /// kerberos_ticket_cache_path of this backend + pub kerberos_ticket_cache_path: Option, + /// user of this backend + pub user: Option, + /// enable the append capacity + pub enable_append: bool, + /// atomic_write_dir of this backend + pub atomic_write_dir: Option, +} + +impl Debug for HdfsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HdfsConfig") + .field("root", &self.root) + .field("name_node", &self.name_node) + .field( + "kerberos_ticket_cache_path", + &self.kerberos_ticket_cache_path, + ) + .field("user", &self.user) + .field("enable_append", &self.enable_append) + .field("atomic_write_dir", &self.atomic_write_dir) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/hdfs/mod.rs b/core/src/services/hdfs/mod.rs index afa3c200a7c3..1b4a4eb77d05 100644 --- a/core/src/services/hdfs/mod.rs +++ b/core/src/services/hdfs/mod.rs @@ -15,9 +15,17 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::HdfsBuilder as Hdfs; -pub use backend::HdfsConfig; +#[cfg(feature = "services-hdfs")] mod lister; +#[cfg(feature = "services-hdfs")] mod reader; +#[cfg(feature = "services-hdfs")] mod writer; + +#[cfg(feature = "services-hdfs")] +mod backend; +#[cfg(feature = "services-hdfs")] +pub use backend::HdfsBuilder as Hdfs; + +mod config; +pub use config::HdfsConfig; diff --git a/core/src/services/hdfs_native/backend.rs b/core/src/services/hdfs_native/backend.rs index d8c9edc3cf64..31ae8ed70ebc 100644 --- a/core/src/services/hdfs_native/backend.rs +++ b/core/src/services/hdfs_native/backend.rs @@ -21,43 +21,18 @@ use std::sync::Arc; use hdfs_native::WriteOptions; use log::debug; -use serde::Deserialize; -use serde::Serialize; -// use uuid::Uuid; use super::error::parse_hdfs_error; use super::lister::HdfsNativeLister; use super::reader::HdfsNativeReader; use super::writer::HdfsNativeWriter; use crate::raw::*; +use crate::services::HdfsNativeConfig; use crate::*; /// [Hadoop Distributed File System (HDFSâ„¢)](https://hadoop.apache.org/) support. /// Using [Native Rust HDFS client](https://github.com/Kimahriman/hdfs-native). -/// Config for HdfsNative services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct HdfsNativeConfig { - /// work dir of this backend - pub root: Option, - /// url of this backend - pub url: Option, - /// enable the append capacity - pub enable_append: bool, -} - -impl Debug for HdfsNativeConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("HdfsNativeConfig") - .field("root", &self.root) - .field("url", &self.url) - .field("enable_append", &self.enable_append) - .finish_non_exhaustive() - } -} - impl Configurator for HdfsNativeConfig { type Builder = HdfsNativeBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/hdfs_native/config.rs b/core/src/services/hdfs_native/config.rs new file mode 100644 index 000000000000..c552c2bf2afe --- /dev/null +++ b/core/src/services/hdfs_native/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for HdfsNative services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct HdfsNativeConfig { + /// work dir of this backend + pub root: Option, + /// url of this backend + pub url: Option, + /// enable the append capacity + pub enable_append: bool, +} + +impl Debug for HdfsNativeConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HdfsNativeConfig") + .field("root", &self.root) + .field("url", &self.url) + .field("enable_append", &self.enable_append) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/hdfs_native/mod.rs b/core/src/services/hdfs_native/mod.rs index a65cacb0a672..0e758428591e 100644 --- a/core/src/services/hdfs_native/mod.rs +++ b/core/src/services/hdfs_native/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::HdfsNativeBuilder as HdfsNative; -pub use backend::HdfsNativeConfig; - +#[cfg(feature = "services-hdfs-native")] mod error; +#[cfg(feature = "services-hdfs-native")] mod lister; +#[cfg(feature = "services-hdfs-native")] mod reader; +#[cfg(feature = "services-hdfs-native")] mod writer; + +#[cfg(feature = "services-hdfs-native")] +mod backend; +#[cfg(feature = "services-hdfs-native")] +pub use backend::HdfsNativeBuilder as HdfsNative; + +mod config; +pub use config::HdfsNativeConfig; diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 5da5e46329a0..7908c2e58fd2 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -26,40 +26,12 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::error::parse_error; use crate::raw::*; +use crate::services::HttpConfig; use crate::*; -/// Config for Http service support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct HttpConfig { - /// endpoint of this backend - pub endpoint: Option, - /// username of this backend - pub username: Option, - /// password of this backend - pub password: Option, - /// token of this backend - pub token: Option, - /// root of this backend - pub root: Option, -} - -impl Debug for HttpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut de = f.debug_struct("HttpConfig"); - de.field("endpoint", &self.endpoint); - de.field("root", &self.root); - - de.finish_non_exhaustive() - } -} - impl Configurator for HttpConfig { type Builder = HttpBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/http/config.rs b/core/src/services/http/config.rs new file mode 100644 index 000000000000..2879ca5efad9 --- /dev/null +++ b/core/src/services/http/config.rs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Http service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct HttpConfig { + /// endpoint of this backend + pub endpoint: Option, + /// username of this backend + pub username: Option, + /// password of this backend + pub password: Option, + /// token of this backend + pub token: Option, + /// root of this backend + pub root: Option, +} + +impl Debug for HttpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("HttpConfig"); + de.field("endpoint", &self.endpoint); + de.field("root", &self.root); + + de.finish_non_exhaustive() + } +} diff --git a/core/src/services/http/mod.rs b/core/src/services/http/mod.rs index 7424d97c2b9b..4e222e9e1a6d 100644 --- a/core/src/services/http/mod.rs +++ b/core/src/services/http/mod.rs @@ -15,8 +15,13 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-http")] +mod error; + +#[cfg(feature = "services-http")] mod backend; +#[cfg(feature = "services-http")] pub use backend::HttpBuilder as Http; -pub use backend::HttpConfig; -mod error; +mod config; +pub use config::HttpConfig; diff --git a/core/src/services/huggingface/backend.rs b/core/src/services/huggingface/backend.rs index 1e9742154336..f0118f1ed1cf 100644 --- a/core/src/services/huggingface/backend.rs +++ b/core/src/services/huggingface/backend.rs @@ -23,69 +23,15 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::HuggingfaceCore; use super::core::HuggingfaceStatus; use super::error::parse_error; use super::lister::HuggingfaceLister; use crate::raw::*; +use crate::services::HuggingfaceConfig; use crate::*; -/// Configuration for Huggingface service support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct HuggingfaceConfig { - /// Repo type of this backend. Default is model. - /// - /// Available values: - /// - model - /// - dataset - pub repo_type: Option, - /// Repo id of this backend. - /// - /// This is required. - pub repo_id: Option, - /// Revision of this backend. - /// - /// Default is main. - pub revision: Option, - /// Root of this backend. Can be "/path/to/dir". - /// - /// Default is "/". - pub root: Option, - /// Token of this backend. - /// - /// This is optional. - pub token: Option, -} - -impl Debug for HuggingfaceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("HuggingfaceConfig"); - - if let Some(repo_type) = &self.repo_type { - ds.field("repo_type", &repo_type); - } - if let Some(repo_id) = &self.repo_id { - ds.field("repo_id", &repo_id); - } - if let Some(revision) = &self.revision { - ds.field("revision", &revision); - } - if let Some(root) = &self.root { - ds.field("root", &root); - } - if self.token.is_some() { - ds.field("token", &""); - } - - ds.finish() - } -} - impl Configurator for HuggingfaceConfig { type Builder = HuggingfaceBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/huggingface/config.rs b/core/src/services/huggingface/config.rs new file mode 100644 index 000000000000..be03c6631179 --- /dev/null +++ b/core/src/services/huggingface/config.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Configuration for Huggingface service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct HuggingfaceConfig { + /// Repo type of this backend. Default is model. + /// + /// Available values: + /// - model + /// - dataset + pub repo_type: Option, + /// Repo id of this backend. + /// + /// This is required. + pub repo_id: Option, + /// Revision of this backend. + /// + /// Default is main. + pub revision: Option, + /// Root of this backend. Can be "/path/to/dir". + /// + /// Default is "/". + pub root: Option, + /// Token of this backend. + /// + /// This is optional. + pub token: Option, +} + +impl Debug for HuggingfaceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("HuggingfaceConfig"); + + if let Some(repo_type) = &self.repo_type { + ds.field("repo_type", &repo_type); + } + if let Some(repo_id) = &self.repo_id { + ds.field("repo_id", &repo_id); + } + if let Some(revision) = &self.revision { + ds.field("revision", &revision); + } + if let Some(root) = &self.root { + ds.field("root", &root); + } + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/huggingface/mod.rs b/core/src/services/huggingface/mod.rs index 3a692fa45e53..011fba47134e 100644 --- a/core/src/services/huggingface/mod.rs +++ b/core/src/services/huggingface/mod.rs @@ -15,10 +15,17 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::HuggingfaceBuilder as Huggingface; -pub use backend::HuggingfaceConfig; - +#[cfg(feature = "services-huggingface")] mod core; +#[cfg(feature = "services-huggingface")] mod error; +#[cfg(feature = "services-huggingface")] mod lister; + +#[cfg(feature = "services-huggingface")] +mod backend; +#[cfg(feature = "services-huggingface")] +pub use backend::HuggingfaceBuilder as Huggingface; + +mod config; +pub use config::HuggingfaceConfig; diff --git a/core/src/services/icloud/backend.rs b/core/src/services/icloud/backend.rs index 583d61a0f0d7..0066946ae824 100644 --- a/core/src/services/icloud/backend.rs +++ b/core/src/services/icloud/backend.rs @@ -21,56 +21,13 @@ use std::sync::Arc; use http::Response; use http::StatusCode; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::Mutex; use super::core::*; use crate::raw::*; +use crate::services::IcloudConfig; use crate::*; -/// Config for icloud services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct IcloudConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - /// - /// default to `/` if not set. - pub root: Option, - /// apple_id of this backend. - /// - /// apple_id must be full, mostly like `example@gmail.com`. - pub apple_id: Option, - /// password of this backend. - /// - /// password must be full. - pub password: Option, - - /// Session - /// - /// token must be valid. - pub trust_token: Option, - /// ds_web_auth_token must be set in Session - pub ds_web_auth_token: Option, - /// enable the china origin - /// China region `origin` Header needs to be set to "https://www.icloud.com.cn". - /// - /// otherwise Apple server will return 302. - pub is_china_mainland: bool, -} - -impl Debug for IcloudConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("IcloudBuilder"); - d.field("root", &self.root); - d.field("is_china_mainland", &self.is_china_mainland); - d.finish_non_exhaustive() - } -} - impl Configurator for IcloudConfig { type Builder = IcloudBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/icloud/config.rs b/core/src/services/icloud/config.rs new file mode 100644 index 000000000000..7fc511457c83 --- /dev/null +++ b/core/src/services/icloud/config.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for icloud services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct IcloudConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// default to `/` if not set. + pub root: Option, + /// apple_id of this backend. + /// + /// apple_id must be full, mostly like `example@gmail.com`. + pub apple_id: Option, + /// password of this backend. + /// + /// password must be full. + pub password: Option, + + /// Session + /// + /// token must be valid. + pub trust_token: Option, + /// ds_web_auth_token must be set in Session + pub ds_web_auth_token: Option, + /// enable the china origin + /// China region `origin` Header needs to be set to "https://www.icloud.com.cn". + /// + /// otherwise Apple server will return 302. + pub is_china_mainland: bool, +} + +impl Debug for IcloudConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("IcloudBuilder"); + d.field("root", &self.root); + d.field("is_china_mainland", &self.is_china_mainland); + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/icloud/mod.rs b/core/src/services/icloud/mod.rs index fd9b28633cb3..92111fee0bf0 100644 --- a/core/src/services/icloud/mod.rs +++ b/core/src/services/icloud/mod.rs @@ -15,8 +15,13 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-icloud")] +mod core; + +#[cfg(feature = "services-icloud")] mod backend; +#[cfg(feature = "services-icloud")] pub use backend::IcloudBuilder as Icloud; -pub use backend::IcloudConfig; -mod core; +mod config; +pub use config::IcloudConfig; diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs index 5589db79a99a..2a72efc9a917 100644 --- a/core/src/services/ipfs/backend.rs +++ b/core/src/services/ipfs/backend.rs @@ -24,25 +24,13 @@ use http::Response; use http::StatusCode; use log::debug; use prost::Message; -use serde::Deserialize; -use serde::Serialize; use super::error::parse_error; use super::ipld::PBNode; use crate::raw::*; +use crate::services::IpfsConfig; use crate::*; -/// Config for IPFS file system support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct IpfsConfig { - /// IPFS gateway endpoint. - pub endpoint: Option, - /// IPFS root. - pub root: Option, -} - impl Configurator for IpfsConfig { type Builder = IpfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/ipfs/config.rs b/core/src/services/ipfs/config.rs new file mode 100644 index 000000000000..2685ccdd957a --- /dev/null +++ b/core/src/services/ipfs/config.rs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for IPFS file system support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct IpfsConfig { + /// IPFS gateway endpoint. + pub endpoint: Option, + /// IPFS root. + pub root: Option, +} diff --git a/core/src/services/ipfs/mod.rs b/core/src/services/ipfs/mod.rs index dfc259740a1e..d12ad5440d02 100644 --- a/core/src/services/ipfs/mod.rs +++ b/core/src/services/ipfs/mod.rs @@ -15,9 +15,15 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-ipfs")] +mod error; +#[cfg(feature = "services-ipfs")] +mod ipld; + +#[cfg(feature = "services-ipfs")] mod backend; +#[cfg(feature = "services-ipfs")] pub use backend::IpfsBuilder as Ipfs; -pub use backend::IpfsConfig; -mod error; -mod ipld; +mod config; +pub use config::IpfsConfig; diff --git a/core/src/services/ipmfs/builder.rs b/core/src/services/ipmfs/builder.rs index 9704888215ca..6bb56ebcde34 100644 --- a/core/src/services/ipmfs/builder.rs +++ b/core/src/services/ipmfs/builder.rs @@ -16,24 +16,12 @@ // under the License. use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::backend::IpmfsBackend; use crate::raw::*; +use crate::services::IpmfsConfig; use crate::*; -/// Config for IPFS MFS support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct IpmfsConfig { - /// Root for ipfs. - pub root: Option, - /// Endpoint for ipfs. - pub endpoint: Option, -} - impl Configurator for IpmfsConfig { type Builder = IpmfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/ipmfs/config.rs b/core/src/services/ipmfs/config.rs new file mode 100644 index 000000000000..f059cdfa10f0 --- /dev/null +++ b/core/src/services/ipmfs/config.rs @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for IPFS MFS support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct IpmfsConfig { + /// Root for ipfs. + pub root: Option, + /// Endpoint for ipfs. + pub endpoint: Option, +} diff --git a/core/src/services/ipmfs/mod.rs b/core/src/services/ipmfs/mod.rs index a054047875eb..b993c242deb0 100644 --- a/core/src/services/ipmfs/mod.rs +++ b/core/src/services/ipmfs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-ipmfs")] mod backend; -mod builder; -pub use builder::IpmfsBuilder as Ipmfs; -pub use builder::IpmfsConfig; - +#[cfg(feature = "services-ipmfs")] mod error; +#[cfg(feature = "services-ipmfs")] mod lister; +#[cfg(feature = "services-ipmfs")] mod writer; + +#[cfg(feature = "services-ipmfs")] +mod builder; +#[cfg(feature = "services-ipmfs")] +pub use builder::IpmfsBuilder as Ipmfs; + +mod config; +pub use config::IpmfsConfig; diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index 7809e35eae0e..0b638df32117 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -23,8 +23,6 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::Mutex; use tokio::sync::OnceCell; @@ -36,36 +34,9 @@ use super::lister::KoofrLister; use super::writer::KoofrWriter; use super::writer::KoofrWriters; use crate::raw::*; +use crate::services::KoofrConfig; use crate::*; -/// Config for Koofr services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct KoofrConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// Koofr endpoint. - pub endpoint: String, - /// Koofr email. - pub email: String, - /// password of this backend. (Must be the application password) - pub password: Option, -} - -impl Debug for KoofrConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Config"); - - ds.field("root", &self.root); - ds.field("email", &self.email); - - ds.finish() - } -} - impl Configurator for KoofrConfig { type Builder = KoofrBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/koofr/config.rs b/core/src/services/koofr/config.rs new file mode 100644 index 000000000000..7ac01c423841 --- /dev/null +++ b/core/src/services/koofr/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Koofr services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct KoofrConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// Koofr endpoint. + pub endpoint: String, + /// Koofr email. + pub email: String, + /// password of this backend. (Must be the application password) + pub password: Option, +} + +impl Debug for KoofrConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Config"); + + ds.field("root", &self.root); + ds.field("email", &self.email); + + ds.finish() + } +} diff --git a/core/src/services/koofr/mod.rs b/core/src/services/koofr/mod.rs index 445d69fbb82e..c73c4dcbe7ff 100644 --- a/core/src/services/koofr/mod.rs +++ b/core/src/services/koofr/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::KoofrBuilder as Koofr; -pub use backend::KoofrConfig; - +#[cfg(feature = "services-koofr")] mod core; +#[cfg(feature = "services-koofr")] mod error; +#[cfg(feature = "services-koofr")] mod lister; +#[cfg(feature = "services-koofr")] mod writer; + +#[cfg(feature = "services-koofr")] +mod backend; +#[cfg(feature = "services-koofr")] +pub use backend::KoofrBuilder as Koofr; + +mod config; +pub use config::KoofrConfig; diff --git a/core/src/services/libsql/backend.rs b/core/src/services/libsql/backend.rs index 8df0c9426635..70ce17889654 100644 --- a/core/src/services/libsql/backend.rs +++ b/core/src/services/libsql/backend.rs @@ -35,51 +35,13 @@ use hrana_client_proto::StmtResult; use hrana_client_proto::Value; use http::Request; use http::Uri; -use serde::Deserialize; -use serde::Serialize; use super::error::parse_error; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::LibsqlConfig; use crate::*; -/// Config for Libsql services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct LibsqlConfig { - /// Connection string for libsql service. - pub connection_string: Option, - /// Authentication token for libsql service. - pub auth_token: Option, - - /// Table name for libsql service. - pub table: Option, - /// Key field name for libsql service. - pub key_field: Option, - /// Value field name for libsql service. - pub value_field: Option, - /// Root for libsql service. - pub root: Option, -} - -impl Debug for LibsqlConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("LibsqlConfig"); - ds.field("connection_string", &self.connection_string) - .field("table", &self.table) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .field("root", &self.root); - - if self.auth_token.is_some() { - ds.field("auth_token", &""); - } - - ds.finish() - } -} - impl Configurator for LibsqlConfig { type Builder = LibsqlBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/libsql/config.rs b/core/src/services/libsql/config.rs new file mode 100644 index 000000000000..fad630d300a1 --- /dev/null +++ b/core/src/services/libsql/config.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Libsql services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct LibsqlConfig { + /// Connection string for libsql service. + pub connection_string: Option, + /// Authentication token for libsql service. + pub auth_token: Option, + + /// Table name for libsql service. + pub table: Option, + /// Key field name for libsql service. + pub key_field: Option, + /// Value field name for libsql service. + pub value_field: Option, + /// Root for libsql service. + pub root: Option, +} + +impl Debug for LibsqlConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("LibsqlConfig"); + ds.field("connection_string", &self.connection_string) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .field("root", &self.root); + + if self.auth_token.is_some() { + ds.field("auth_token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/libsql/mod.rs b/core/src/services/libsql/mod.rs index 086dc5910900..7e859d5fdb49 100644 --- a/core/src/services/libsql/mod.rs +++ b/core/src/services/libsql/mod.rs @@ -15,8 +15,13 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-libsql")] +mod error; + +#[cfg(feature = "services-libsql")] mod backend; +#[cfg(feature = "services-libsql")] pub use backend::LibsqlBuilder as Libsql; -pub use backend::LibsqlConfig; -mod error; +mod config; +pub use config::LibsqlConfig; diff --git a/core/src/services/memcached/backend.rs b/core/src/services/memcached/backend.rs index 2e8526c0cbf7..c89b81ba704a 100644 --- a/core/src/services/memcached/backend.rs +++ b/core/src/services/memcached/backend.rs @@ -18,37 +18,15 @@ use std::time::Duration; use bb8::RunError; -use serde::Deserialize; -use serde::Serialize; use tokio::net::TcpStream; use tokio::sync::OnceCell; use super::binary; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::MemcachedConfig; use crate::*; -/// Config for MemCached services support -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MemcachedConfig { - /// network address of the memcached service. - /// - /// For example: "tcp://localhost:11211" - pub endpoint: Option, - /// the working directory of the service. Can be "/path/to/dir" - /// - /// default is "/" - pub root: Option, - /// Memcached username, optional. - pub username: Option, - /// Memcached password, optional. - pub password: Option, - /// The default ttl for put operations. - pub default_ttl: Option, -} - impl Configurator for MemcachedConfig { type Builder = MemcachedBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/memcached/config.rs b/core/src/services/memcached/config.rs new file mode 100644 index 000000000000..c05a667cb519 --- /dev/null +++ b/core/src/services/memcached/config.rs @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; +use std::time::Duration; + +/// Config for MemCached services support +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MemcachedConfig { + /// network address of the memcached service. + /// + /// For example: "tcp://localhost:11211" + pub endpoint: Option, + /// the working directory of the service. Can be "/path/to/dir" + /// + /// default is "/" + pub root: Option, + /// Memcached username, optional. + pub username: Option, + /// Memcached password, optional. + pub password: Option, + /// The default ttl for put operations. + pub default_ttl: Option, +} diff --git a/core/src/services/memcached/mod.rs b/core/src/services/memcached/mod.rs index d293520797cb..3065eea73587 100644 --- a/core/src/services/memcached/mod.rs +++ b/core/src/services/memcached/mod.rs @@ -15,7 +15,13 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-memcached")] +mod binary; + +#[cfg(feature = "services-memcached")] mod backend; +#[cfg(feature = "services-memcached")] pub use backend::MemcachedBuilder as Memcached; -pub use backend::MemcachedConfig; -mod binary; + +mod config; +pub use config::MemcachedConfig; diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs index ef7ca83760fc..93041ba51697 100644 --- a/core/src/services/memory/backend.rs +++ b/core/src/services/memory/backend.rs @@ -20,22 +20,11 @@ use std::fmt::Debug; use std::sync::Arc; use std::sync::Mutex; -use serde::Deserialize; -use serde::Serialize; - use crate::raw::adapters::typed_kv; use crate::raw::Access; +use crate::services::MemoryConfig; use crate::*; -/// Config for memory. -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MemoryConfig { - /// root of the backend. - pub root: Option, -} - impl Configurator for MemoryConfig { type Builder = MemoryBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/memory/config.rs b/core/src/services/memory/config.rs new file mode 100644 index 000000000000..e9d5a97abb3d --- /dev/null +++ b/core/src/services/memory/config.rs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for memory. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MemoryConfig { + /// root of the backend. + pub root: Option, +} diff --git a/core/src/services/memory/mod.rs b/core/src/services/memory/mod.rs index 6ba9c2bac330..ecf2e8d27d21 100644 --- a/core/src/services/memory/mod.rs +++ b/core/src/services/memory/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-memory")] mod backend; +#[cfg(feature = "services-memory")] pub use backend::MemoryBuilder as Memory; -pub use backend::MemoryConfig; + +mod config; +pub use config::MemoryConfig; diff --git a/core/src/services/mini_moka/backend.rs b/core/src/services/mini_moka/backend.rs index e84a82c8f38e..f99bf0a1ecbb 100644 --- a/core/src/services/mini_moka/backend.rs +++ b/core/src/services/mini_moka/backend.rs @@ -21,35 +21,12 @@ use std::time::Duration; use log::debug; use mini_moka::sync::Cache; use mini_moka::sync::CacheBuilder; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::typed_kv; use crate::raw::Access; +use crate::services::MiniMokaConfig; use crate::*; -/// Config for mini-moka support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MiniMokaConfig { - /// Sets the max capacity of the cache. - /// - /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity) - pub max_capacity: Option, - /// Sets the time to live of the cache. - /// - /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live) - pub time_to_live: Option, - /// Sets the time to idle of the cache. - /// - /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle) - pub time_to_idle: Option, - - /// root path of this backend - pub root: Option, -} - impl Configurator for MiniMokaConfig { type Builder = MiniMokaBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/mini_moka/config.rs b/core/src/services/mini_moka/config.rs new file mode 100644 index 000000000000..ff76b1ae005b --- /dev/null +++ b/core/src/services/mini_moka/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; +use std::time::Duration; + +/// Config for mini-moka support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MiniMokaConfig { + /// Sets the max capacity of the cache. + /// + /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity) + pub max_capacity: Option, + /// Sets the time to live of the cache. + /// + /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live) + pub time_to_live: Option, + /// Sets the time to idle of the cache. + /// + /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle) + pub time_to_idle: Option, + + /// root path of this backend + pub root: Option, +} diff --git a/core/src/services/mini_moka/mod.rs b/core/src/services/mini_moka/mod.rs index cef6a0630b07..bb3b141aa24e 100644 --- a/core/src/services/mini_moka/mod.rs +++ b/core/src/services/mini_moka/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-mini-moka")] mod backend; +#[cfg(feature = "services-mini-moka")] pub use backend::MiniMokaBuilder as MiniMoka; -pub use backend::MiniMokaConfig; + +mod config; +pub use config::MiniMokaConfig; diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index 10e9aa7f2e70..62ece326329d 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -19,457 +19,197 @@ //! //! More ongoing services support is tracked at [opendal#5](https://github.com/apache/opendal/issues/5). Please feel free to submit issues if there are services not covered. -#[cfg(feature = "services-aliyun-drive")] mod aliyun_drive; -#[cfg(feature = "services-aliyun-drive")] -pub use aliyun_drive::AliyunDrive; -#[cfg(feature = "services-aliyun-drive")] -pub use aliyun_drive::AliyunDriveConfig; +pub use aliyun_drive::*; + +mod alluxio; +pub use alluxio::*; + +mod atomicserver; +pub use self::atomicserver::*; -#[cfg(feature = "services-azblob")] mod azblob; -#[cfg(feature = "services-azblob")] -pub use azblob::Azblob; -#[cfg(feature = "services-azblob")] -pub use azblob::AzblobConfig; +pub use azblob::*; -#[cfg(feature = "services-azdls")] mod azdls; -#[cfg(feature = "services-azdls")] -pub use azdls::Azdls; -#[cfg(feature = "services-azdls")] -pub use azdls::AzdlsConfig; +pub use azdls::*; + +mod azfile; +pub use azfile::*; + +mod b2; +pub use b2::*; + +mod cacache; +pub use self::cacache::*; + +mod chainsafe; +pub use chainsafe::*; -#[cfg(feature = "services-cloudflare-kv")] mod cloudflare_kv; -#[cfg(feature = "services-cloudflare-kv")] -pub use self::cloudflare_kv::CloudflareKv; -#[cfg(feature = "services-cloudflare-kv")] -pub use self::cloudflare_kv::CloudflareKvConfig; +pub use self::cloudflare_kv::*; + +mod compfs; +pub use compfs::*; -#[cfg(feature = "services-cos")] mod cos; -#[cfg(feature = "services-cos")] -pub use cos::Cos; -#[cfg(feature = "services-cos")] -pub use cos::CosConfig; +pub use cos::*; + +mod d1; +pub use self::d1::*; -#[cfg(feature = "services-dashmap")] mod dashmap; -#[cfg(feature = "services-dashmap")] -pub use self::dashmap::Dashmap; -#[cfg(feature = "services-dashmap")] -pub use self::dashmap::DashmapConfig; +pub use self::dashmap::*; + +mod dbfs; +pub use self::dbfs::*; + +mod dropbox; +pub use dropbox::*; -#[cfg(feature = "services-etcd")] mod etcd; -#[cfg(feature = "services-etcd")] -pub use self::etcd::Etcd; -#[cfg(feature = "services-etcd")] -pub use self::etcd::EtcdConfig; +pub use self::etcd::*; + +mod foundationdb; +pub use self::foundationdb::*; -#[cfg(feature = "services-fs")] mod fs; -#[cfg(feature = "services-fs")] -pub use fs::Fs; -#[cfg(feature = "services-fs")] -pub use fs::FsConfig; +pub use fs::*; -#[cfg(feature = "services-ftp")] mod ftp; -#[cfg(feature = "services-ftp")] -pub use ftp::Ftp; -#[cfg(feature = "services-ftp")] -pub use ftp::FtpConfig; +pub use ftp::*; -#[cfg(feature = "services-gcs")] mod gcs; -#[cfg(feature = "services-gcs")] -pub use gcs::Gcs; -#[cfg(feature = "services-gcs")] -pub use gcs::GcsConfig; +pub use gcs::*; + +mod gdrive; +pub use gdrive::*; -#[cfg(feature = "services-ghac")] mod ghac; -#[cfg(feature = "services-ghac")] -pub use ghac::Ghac; -#[cfg(feature = "services-ghac")] -pub use ghac::GhacConfig; +pub use ghac::*; + +mod github; +pub use github::*; -#[cfg(feature = "services-gridfs")] mod gridfs; -#[cfg(feature = "services-gridfs")] -pub use gridfs::GridFsConfig; -#[cfg(feature = "services-gridfs")] -pub use gridfs::Gridfs; +pub use gridfs::*; -#[cfg(feature = "services-hdfs")] mod hdfs; -#[cfg(feature = "services-hdfs")] -pub use self::hdfs::Hdfs; -#[cfg(feature = "services-hdfs")] -pub use self::hdfs::HdfsConfig; +pub use self::hdfs::*; + +mod hdfs_native; +pub use hdfs_native::*; -#[cfg(feature = "services-http")] mod http; -#[cfg(feature = "services-http")] -pub use self::http::Http; -#[cfg(feature = "services-http")] -pub use self::http::HttpConfig; +pub use self::http::*; -#[cfg(feature = "services-huggingface")] mod huggingface; -#[cfg(feature = "services-huggingface")] -pub use huggingface::Huggingface; -#[cfg(feature = "services-huggingface")] -pub use huggingface::HuggingfaceConfig; +pub use huggingface::*; + +mod icloud; +pub use icloud::*; -#[cfg(feature = "services-ipfs")] mod ipfs; -#[cfg(feature = "services-ipfs")] -pub use self::ipfs::Ipfs; -#[cfg(feature = "services-ipfs")] -pub use self::ipfs::IpfsConfig; +pub use self::ipfs::*; -#[cfg(feature = "services-ipmfs")] mod ipmfs; -#[cfg(feature = "services-ipmfs")] -pub use ipmfs::Ipmfs; -#[cfg(feature = "services-ipmfs")] -pub use ipmfs::IpmfsConfig; +pub use ipmfs::*; -#[cfg(feature = "services-icloud")] -mod icloud; -#[cfg(feature = "services-icloud")] -pub use icloud::Icloud; -#[cfg(feature = "services-icloud")] -pub use icloud::IcloudConfig; +mod koofr; +pub use koofr::*; -#[cfg(feature = "services-libsql")] mod libsql; -#[cfg(feature = "services-libsql")] -pub use libsql::Libsql; -#[cfg(feature = "services-libsql")] -pub use libsql::LibsqlConfig; +pub use libsql::*; -#[cfg(feature = "services-memcached")] mod memcached; -#[cfg(feature = "services-memcached")] -pub use memcached::Memcached; -#[cfg(feature = "services-memcached")] -pub use memcached::MemcachedConfig; +pub use memcached::*; -#[cfg(feature = "services-memory")] mod memory; -#[cfg(feature = "services-memory")] -pub use self::memory::Memory; -#[cfg(feature = "services-memory")] -pub use self::memory::MemoryConfig; +pub use self::memory::*; -#[cfg(feature = "services-mini-moka")] mod mini_moka; -#[cfg(feature = "services-mini-moka")] -pub use self::mini_moka::MiniMoka; -#[cfg(feature = "services-mini-moka")] -pub use self::mini_moka::MiniMokaConfig; +pub use self::mini_moka::*; -#[cfg(feature = "services-moka")] mod moka; -#[cfg(feature = "services-moka")] -pub use self::moka::Moka; -#[cfg(feature = "services-moka")] -pub use self::moka::MokaConfig; - -#[cfg(feature = "services-obs")] -mod obs; -#[cfg(feature = "services-obs")] -pub use obs::Obs; -#[cfg(feature = "services-obs")] -pub use obs::ObsConfig; - -#[cfg(feature = "services-oss")] -mod oss; -#[cfg(feature = "services-oss")] -pub use oss::Oss; -#[cfg(feature = "services-oss")] -pub use oss::OssConfig; - -#[cfg(feature = "services-cacache")] -mod cacache; -#[cfg(feature = "services-cacache")] -pub use self::cacache::Cacache; -#[cfg(feature = "services-cacache")] -pub use self::cacache::CacacheConfig; - -#[cfg(feature = "services-persy")] -mod persy; -#[cfg(feature = "services-persy")] -pub use self::persy::Persy; -#[cfg(feature = "services-persy")] -pub use self::persy::PersyConfig; +pub use self::moka::*; -#[cfg(feature = "services-redis")] -mod redis; -#[cfg(feature = "services-redis")] -pub use self::redis::Redis; -#[cfg(feature = "services-redis")] -pub use self::redis::RedisConfig; - -#[cfg(feature = "services-rocksdb")] -mod rocksdb; -#[cfg(feature = "services-rocksdb")] -pub use self::rocksdb::Rocksdb; -#[cfg(feature = "services-rocksdb")] -pub use self::rocksdb::RocksdbConfig; - -#[cfg(feature = "services-s3")] -mod s3; -#[cfg(feature = "services-s3")] -pub use s3::S3Config; -#[cfg(feature = "services-s3")] -pub use s3::S3; - -#[cfg(feature = "services-sftp")] -mod sftp; -#[cfg(feature = "services-sftp")] -pub use sftp::Sftp; -#[cfg(feature = "services-sftp")] -pub use sftp::SftpConfig; - -#[cfg(feature = "services-sled")] -mod sled; -#[cfg(feature = "services-sled")] -pub use self::sled::Sled; -#[cfg(feature = "services-sled")] -pub use self::sled::SledConfig; +mod mongodb; +pub use self::mongodb::*; -#[cfg(feature = "services-supabase")] -mod supabase; -#[cfg(feature = "services-supabase")] -pub use supabase::Supabase; -#[cfg(feature = "services-supabase")] -pub use supabase::SupabaseConfig; +mod monoiofs; +pub use monoiofs::*; -#[cfg(feature = "services-webdav")] -mod webdav; -#[cfg(feature = "services-webdav")] -pub use webdav::Webdav; -#[cfg(feature = "services-webdav")] -pub use webdav::WebdavConfig; +mod mysql; +pub use self::mysql::*; -#[cfg(feature = "services-webhdfs")] -mod webhdfs; -#[cfg(feature = "services-webhdfs")] -pub use webhdfs::Webhdfs; -#[cfg(feature = "services-webhdfs")] -pub use webhdfs::WebhdfsConfig; +mod obs; +pub use obs::*; -#[cfg(feature = "services-onedrive")] mod onedrive; -#[cfg(feature = "services-onedrive")] -pub use onedrive::Onedrive; -#[cfg(feature = "services-onedrive")] -pub use onedrive::OnedriveConfig; +pub use onedrive::*; -#[cfg(feature = "services-gdrive")] -mod gdrive; -#[cfg(feature = "services-gdrive")] -pub use gdrive::Gdrive; -#[cfg(feature = "services-gdrive")] -pub use gdrive::GdriveConfig; +mod oss; +pub use oss::*; -#[cfg(feature = "services-github")] -mod github; -#[cfg(feature = "services-github")] -pub use github::Github; -#[cfg(feature = "services-github")] -pub use github::GithubConfig; +mod pcloud; +pub use pcloud::*; -#[cfg(feature = "services-dropbox")] -mod dropbox; -#[cfg(feature = "services-dropbox")] -pub use dropbox::Dropbox; -#[cfg(feature = "services-dropbox")] -pub use dropbox::DropboxConfig; +mod persy; +pub use self::persy::*; -#[cfg(feature = "services-vercel-artifacts")] -mod vercel_artifacts; -#[cfg(feature = "services-vercel-artifacts")] -pub use vercel_artifacts::VercelArtifacts; -#[cfg(feature = "services-vercel-artifacts")] -pub use vercel_artifacts::VercelArtifactsConfig; +mod postgresql; +pub use self::postgresql::*; -#[cfg(feature = "services-redb")] mod redb; -#[cfg(feature = "services-redb")] -pub use self::redb::Redb; -#[cfg(feature = "services-redb")] -pub use self::redb::RedbConfig; +pub use self::redb::*; -#[cfg(feature = "services-tikv")] -mod tikv; -#[cfg(feature = "services-tikv")] -pub use self::tikv::Tikv; -#[cfg(feature = "services-tikv")] -pub use self::tikv::TikvConfig; - -#[cfg(feature = "services-foundationdb")] -mod foundationdb; -#[cfg(feature = "services-foundationdb")] -pub use self::foundationdb::Foundationdb; -#[cfg(feature = "services-foundationdb")] -pub use self::foundationdb::FoundationdbConfig; +mod redis; +pub use self::redis::*; -#[cfg(feature = "services-postgresql")] -mod postgresql; -#[cfg(feature = "services-postgresql")] -pub use self::postgresql::Postgresql; -#[cfg(feature = "services-postgresql")] -pub use self::postgresql::PostgresqlConfig; +mod rocksdb; +pub use self::rocksdb::*; -#[cfg(feature = "services-atomicserver")] -mod atomicserver; -#[cfg(feature = "services-atomicserver")] -pub use self::atomicserver::Atomicserver; -#[cfg(feature = "services-atomicserver")] -pub use self::atomicserver::AtomicserverConfig; +mod s3; +pub use s3::*; -#[cfg(feature = "services-mysql")] -mod mysql; -#[cfg(feature = "services-mysql")] -pub use self::mysql::Mysql; -#[cfg(feature = "services-mysql")] -pub use self::mysql::MysqlConfig; +mod seafile; +pub use seafile::*; -#[cfg(feature = "services-sqlite")] -mod sqlite; -#[cfg(feature = "services-sqlite")] -pub use sqlite::Sqlite; -#[cfg(feature = "services-sqlite")] -pub use sqlite::SqliteConfig; +mod sftp; +pub use sftp::*; -#[cfg(feature = "services-d1")] -mod d1; -#[cfg(feature = "services-d1")] -pub use self::d1::D1Config; -#[cfg(feature = "services-d1")] -pub use self::d1::D1; +mod sled; +pub use self::sled::*; -#[cfg(feature = "services-azfile")] -mod azfile; -#[cfg(feature = "services-azfile")] -pub use self::azfile::Azfile; -#[cfg(feature = "services-azfile")] -pub use self::azfile::AzfileConfig; +mod sqlite; +pub use self::sqlite::*; -#[cfg(feature = "services-mongodb")] -mod mongodb; -#[cfg(feature = "services-mongodb")] -pub use self::mongodb::Mongodb; -#[cfg(feature = "services-mongodb")] -pub use self::mongodb::MongodbConfig; +mod supabase; +pub use supabase::*; -#[cfg(feature = "services-dbfs")] -mod dbfs; -#[cfg(feature = "services-dbfs")] -pub use self::dbfs::Dbfs; -#[cfg(feature = "services-dbfs")] -pub use self::dbfs::DbfsConfig; +mod surrealdb; +pub use surrealdb::*; -#[cfg(feature = "services-swift")] mod swift; -#[cfg(feature = "services-swift")] -pub use self::swift::Swift; -#[cfg(feature = "services-swift")] -pub use self::swift::SwiftConfig; +pub use self::swift::*; -#[cfg(feature = "services-alluxio")] -mod alluxio; -#[cfg(feature = "services-alluxio")] -pub use alluxio::Alluxio; -#[cfg(feature = "services-alluxio")] -pub use alluxio::AlluxioConfig; - -#[cfg(feature = "services-b2")] -mod b2; -#[cfg(feature = "services-b2")] -pub use b2::B2Config; -#[cfg(feature = "services-b2")] -pub use b2::B2; - -#[cfg(feature = "services-seafile")] -mod seafile; -#[cfg(feature = "services-seafile")] -pub use seafile::Seafile; -#[cfg(feature = "services-seafile")] -pub use seafile::SeafileConfig; +mod tikv; +pub use self::tikv::*; -#[cfg(feature = "services-upyun")] mod upyun; -#[cfg(feature = "services-upyun")] -pub use upyun::Upyun; -#[cfg(feature = "services-upyun")] -pub use upyun::UpyunConfig; - -#[cfg(feature = "services-chainsafe")] -mod chainsafe; -#[cfg(feature = "services-chainsafe")] -pub use chainsafe::Chainsafe; -#[cfg(feature = "services-chainsafe")] -pub use chainsafe::ChainsafeConfig; - -#[cfg(feature = "services-pcloud")] -mod pcloud; -#[cfg(feature = "services-pcloud")] -pub use pcloud::Pcloud; -#[cfg(feature = "services-pcloud")] -pub use pcloud::PcloudConfig; - -#[cfg(feature = "services-hdfs-native")] -mod hdfs_native; -#[cfg(feature = "services-hdfs-native")] -pub use hdfs_native::HdfsNative; -#[cfg(feature = "services-hdfs-native")] -pub use hdfs_native::HdfsNativeConfig; +pub use upyun::*; -#[cfg(feature = "services-yandex-disk")] -mod yandex_disk; -#[cfg(feature = "services-yandex-disk")] -pub use yandex_disk::YandexDisk; -#[cfg(feature = "services-yandex-disk")] -pub use yandex_disk::YandexDiskConfig; - -#[cfg(feature = "services-koofr")] -mod koofr; -#[cfg(feature = "services-koofr")] -pub use koofr::Koofr; -#[cfg(feature = "services-koofr")] -pub use koofr::KoofrConfig; +mod vercel_artifacts; +pub use vercel_artifacts::*; -#[cfg(feature = "services-vercel-blob")] mod vercel_blob; -#[cfg(feature = "services-vercel-blob")] -pub use vercel_blob::VercelBlob; -#[cfg(feature = "services-vercel-blob")] -pub use vercel_blob::VercelBlobConfig; +pub use vercel_blob::*; -#[cfg(feature = "services-surrealdb")] -mod surrealdb; -#[cfg(feature = "services-surrealdb")] -pub use surrealdb::Surrealdb; -#[cfg(feature = "services-surrealdb")] -pub use surrealdb::SurrealdbConfig; +mod webdav; +pub use webdav::*; -#[cfg(feature = "services-compfs")] -mod compfs; -#[cfg(feature = "services-compfs")] -pub use compfs::Compfs; -#[cfg(feature = "services-compfs")] -pub use compfs::CompfsConfig; +mod webhdfs; +pub use webhdfs::*; -#[cfg(feature = "services-monoiofs")] -mod monoiofs; -#[cfg(feature = "services-monoiofs")] -pub use monoiofs::Monoiofs; -#[cfg(feature = "services-monoiofs")] -pub use monoiofs::MonoiofsConfig; +mod yandex_disk; +pub use yandex_disk::*; diff --git a/core/src/services/moka/backend.rs b/core/src/services/moka/backend.rs index 1f29fd8bcf87..451bb74c5031 100644 --- a/core/src/services/moka/backend.rs +++ b/core/src/services/moka/backend.rs @@ -22,54 +22,12 @@ use std::time::Duration; use log::debug; use moka::sync::CacheBuilder; use moka::sync::SegmentedCache; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::typed_kv; use crate::raw::*; +use crate::services::MokaConfig; use crate::*; -/// Config for Moka services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MokaConfig { - /// Name for this cache instance. - pub name: Option, - /// Sets the max capacity of the cache. - /// - /// Refer to [`moka::sync::CacheBuilder::max_capacity`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.max_capacity) - pub max_capacity: Option, - /// Sets the time to live of the cache. - /// - /// Refer to [`moka::sync::CacheBuilder::time_to_live`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.time_to_live) - pub time_to_live: Option, - /// Sets the time to idle of the cache. - /// - /// Refer to [`moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.time_to_idle) - pub time_to_idle: Option, - /// Sets the segments number of the cache. - /// - /// Refer to [`moka::sync::CacheBuilder::segments`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.segments) - pub num_segments: Option, - - /// root path of this backend - pub root: Option, -} - -impl Debug for MokaConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MokaConfig") - .field("name", &self.name) - .field("max_capacity", &self.max_capacity) - .field("time_to_live", &self.time_to_live) - .field("time_to_idle", &self.time_to_idle) - .field("num_segments", &self.num_segments) - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for MokaConfig { type Builder = MokaBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/moka/config.rs b/core/src/services/moka/config.rs new file mode 100644 index 000000000000..8e0976456be0 --- /dev/null +++ b/core/src/services/moka/config.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; +use std::time::Duration; + +/// Config for Moka services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MokaConfig { + /// Name for this cache instance. + pub name: Option, + /// Sets the max capacity of the cache. + /// + /// Refer to [`moka::sync::CacheBuilder::max_capacity`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.max_capacity) + pub max_capacity: Option, + /// Sets the time to live of the cache. + /// + /// Refer to [`moka::sync::CacheBuilder::time_to_live`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.time_to_live) + pub time_to_live: Option, + /// Sets the time to idle of the cache. + /// + /// Refer to [`moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.time_to_idle) + pub time_to_idle: Option, + /// Sets the segments number of the cache. + /// + /// Refer to [`moka::sync::CacheBuilder::segments`](https://docs.rs/moka/latest/moka/sync/struct.CacheBuilder.html#method.segments) + pub num_segments: Option, + + /// root path of this backend + pub root: Option, +} + +impl Debug for MokaConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MokaConfig") + .field("name", &self.name) + .field("max_capacity", &self.max_capacity) + .field("time_to_live", &self.time_to_live) + .field("time_to_idle", &self.time_to_idle) + .field("num_segments", &self.num_segments) + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/moka/mod.rs b/core/src/services/moka/mod.rs index 0d79a37bef1b..89605e7ca4b4 100644 --- a/core/src/services/moka/mod.rs +++ b/core/src/services/moka/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-moka")] mod backend; +#[cfg(feature = "services-moka")] pub use backend::MokaBuilder as Moka; -pub use backend::MokaConfig; + +mod config; +pub use config::MokaConfig; diff --git a/core/src/services/mongodb/backend.rs b/core/src/services/mongodb/backend.rs index 8c7cb893c157..24abe6fb03ef 100644 --- a/core/src/services/mongodb/backend.rs +++ b/core/src/services/mongodb/backend.rs @@ -22,46 +22,13 @@ use mongodb::bson::doc; use mongodb::bson::Binary; use mongodb::bson::Document; use mongodb::options::ClientOptions; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::MongodbConfig; use crate::*; -/// Config for Mongodb service support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MongodbConfig { - /// connection string of this backend - pub connection_string: Option, - /// database of this backend - pub database: Option, - /// collection of this backend - pub collection: Option, - /// root of this backend - pub root: Option, - /// key field of this backend - pub key_field: Option, - /// value field of this backend - pub value_field: Option, -} - -impl Debug for MongodbConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MongodbConfig") - .field("connection_string", &self.connection_string) - .field("database", &self.database) - .field("collection", &self.collection) - .field("root", &self.root) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .finish() - } -} - impl Configurator for MongodbConfig { type Builder = MongodbBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/mongodb/config.rs b/core/src/services/mongodb/config.rs new file mode 100644 index 000000000000..a20662ed453a --- /dev/null +++ b/core/src/services/mongodb/config.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Mongodb service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MongodbConfig { + /// connection string of this backend + pub connection_string: Option, + /// database of this backend + pub database: Option, + /// collection of this backend + pub collection: Option, + /// root of this backend + pub root: Option, + /// key field of this backend + pub key_field: Option, + /// value field of this backend + pub value_field: Option, +} + +impl Debug for MongodbConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MongodbConfig") + .field("connection_string", &self.connection_string) + .field("database", &self.database) + .field("collection", &self.collection) + .field("root", &self.root) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .finish() + } +} diff --git a/core/src/services/mongodb/mod.rs b/core/src/services/mongodb/mod.rs index df5a7a3fc336..a276de95db2e 100644 --- a/core/src/services/mongodb/mod.rs +++ b/core/src/services/mongodb/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-mongodb")] mod backend; +#[cfg(feature = "services-mongodb")] pub use backend::MongodbBuilder as Mongodb; -pub use backend::MongodbConfig; + +mod config; +pub use config::MongodbConfig; diff --git a/core/src/services/monoiofs/backend.rs b/core/src/services/monoiofs/backend.rs index c2443f148b95..89c5b0ebdc30 100644 --- a/core/src/services/monoiofs/backend.rs +++ b/core/src/services/monoiofs/backend.rs @@ -21,28 +21,14 @@ use std::path::PathBuf; use std::sync::Arc; use chrono::DateTime; -use serde::Deserialize; -use serde::Serialize; use super::core::MonoiofsCore; use super::reader::MonoiofsReader; use super::writer::MonoiofsWriter; use crate::raw::*; +use crate::services::MonoiofsConfig; use crate::*; -/// Config for monoiofs services support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MonoiofsConfig { - /// The Root of this backend. - /// - /// All operations will happen under this root. - /// - /// Builder::build will return error if not set. - pub root: Option, -} - impl Configurator for MonoiofsConfig { type Builder = MonoiofsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/monoiofs/config.rs b/core/src/services/monoiofs/config.rs new file mode 100644 index 000000000000..ea55ae58fc18 --- /dev/null +++ b/core/src/services/monoiofs/config.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for monoiofs services support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MonoiofsConfig { + /// The Root of this backend. + /// + /// All operations will happen under this root. + /// + /// Builder::build will return error if not set. + pub root: Option, +} diff --git a/core/src/services/monoiofs/mod.rs b/core/src/services/monoiofs/mod.rs index dbda5ac14786..e342397581c6 100644 --- a/core/src/services/monoiofs/mod.rs +++ b/core/src/services/monoiofs/mod.rs @@ -15,10 +15,17 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::MonoiofsBuilder as Monoiofs; -pub use backend::MonoiofsConfig; - +#[cfg(feature = "services-monoiofs")] mod core; +#[cfg(feature = "services-monoiofs")] mod reader; +#[cfg(feature = "services-monoiofs")] mod writer; + +#[cfg(feature = "services-monoiofs")] +mod backend; +#[cfg(feature = "services-monoiofs")] +pub use backend::MonoiofsBuilder as Monoiofs; + +mod config; +pub use config::MonoiofsConfig; diff --git a/core/src/services/mysql/backend.rs b/core/src/services/mysql/backend.rs index cdf9c90b6893..9e55d02f6416 100644 --- a/core/src/services/mysql/backend.rs +++ b/core/src/services/mysql/backend.rs @@ -20,47 +20,12 @@ use std::fmt::Debug; use mysql_async::prelude::*; use mysql_async::Opts; use mysql_async::Pool; -use serde::Deserialize; -use serde::Serialize; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::MysqlConfig; use crate::*; -/// Config for Mysql services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct MysqlConfig { - /// The connection string for mysql. - pub connection_string: Option, - - /// The table name for mysql. - pub table: Option, - /// The key field name for mysql. - pub key_field: Option, - /// The value field name for mysql. - pub value_field: Option, - /// The root for mysql. - pub root: Option, -} - -impl Debug for MysqlConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("MysqlConfig"); - - if self.connection_string.is_some() { - d.field("connection_string", &""); - } - - d.field("root", &self.root) - .field("table", &self.table) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .finish() - } -} - impl Configurator for MysqlConfig { type Builder = MysqlBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/mysql/config.rs b/core/src/services/mysql/config.rs new file mode 100644 index 000000000000..a27037b93509 --- /dev/null +++ b/core/src/services/mysql/config.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Mysql services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct MysqlConfig { + /// The connection string for mysql. + pub connection_string: Option, + + /// The table name for mysql. + pub table: Option, + /// The key field name for mysql. + pub key_field: Option, + /// The value field name for mysql. + pub value_field: Option, + /// The root for mysql. + pub root: Option, +} + +impl Debug for MysqlConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("MysqlConfig"); + + if self.connection_string.is_some() { + d.field("connection_string", &""); + } + + d.field("root", &self.root) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .finish() + } +} diff --git a/core/src/services/mysql/mod.rs b/core/src/services/mysql/mod.rs index fd4ce87a676e..9b333889429b 100644 --- a/core/src/services/mysql/mod.rs +++ b/core/src/services/mysql/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-mysql")] mod backend; +#[cfg(feature = "services-mysql")] pub use backend::MysqlBuilder as Mysql; -pub use backend::MysqlConfig; + +mod config; +pub use config::MysqlConfig; diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 20433c8854cb..b84c62e77085 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -26,8 +26,6 @@ use log::debug; use reqsign::HuaweicloudObsConfig; use reqsign::HuaweicloudObsCredentialLoader; use reqsign::HuaweicloudObsSigner; -use serde::Deserialize; -use serde::Serialize; use super::core::ObsCore; use super::error::parse_error; @@ -35,37 +33,9 @@ use super::lister::ObsLister; use super::writer::ObsWriter; use crate::raw::*; use crate::services::obs::writer::ObsWriters; +use crate::services::ObsConfig; use crate::*; -/// Config for Huawei-Cloud Object Storage Service (OBS) support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct ObsConfig { - /// Root for obs. - pub root: Option, - /// Endpoint for obs. - pub endpoint: Option, - /// Access key id for obs. - pub access_key_id: Option, - /// Secret access key for obs. - pub secret_access_key: Option, - /// Bucket for obs. - pub bucket: Option, -} - -impl Debug for ObsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ObsConfig") - .field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("access_key_id", &"") - .field("secret_access_key", &"") - .field("bucket", &self.bucket) - .finish() - } -} - impl Configurator for ObsConfig { type Builder = ObsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/obs/config.rs b/core/src/services/obs/config.rs new file mode 100644 index 000000000000..5c7a1667cf02 --- /dev/null +++ b/core/src/services/obs/config.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Huawei-Cloud Object Storage Service (OBS) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct ObsConfig { + /// Root for obs. + pub root: Option, + /// Endpoint for obs. + pub endpoint: Option, + /// Access key id for obs. + pub access_key_id: Option, + /// Secret access key for obs. + pub secret_access_key: Option, + /// Bucket for obs. + pub bucket: Option, +} + +impl Debug for ObsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObsConfig") + .field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("access_key_id", &"") + .field("secret_access_key", &"") + .field("bucket", &self.bucket) + .finish() + } +} diff --git a/core/src/services/obs/mod.rs b/core/src/services/obs/mod.rs index daa275138171..612a526889e0 100644 --- a/core/src/services/obs/mod.rs +++ b/core/src/services/obs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::ObsBuilder as Obs; -pub use backend::ObsConfig; - +#[cfg(feature = "services-obs")] mod core; +#[cfg(feature = "services-obs")] mod error; +#[cfg(feature = "services-obs")] mod lister; +#[cfg(feature = "services-obs")] mod writer; + +#[cfg(feature = "services-obs")] +mod backend; +#[cfg(feature = "services-obs")] +pub use backend::ObsBuilder as Obs; + +mod config; +pub use config::ObsConfig; diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 5b3fb36739bd..9ee0eb0e989e 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -19,35 +19,15 @@ use std::fmt::Debug; use std::fmt::Formatter; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::backend::OnedriveBackend; use crate::raw::normalize_root; use crate::raw::Access; use crate::raw::HttpClient; +use crate::services::OnedriveConfig; use crate::Scheme; use crate::*; -/// Config for [OneDrive](https://onedrive.com) backend support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct OnedriveConfig { - /// bearer access token for OneDrive - pub access_token: Option, - /// root path of OneDrive folder. - pub root: Option, -} - -impl Debug for OnedriveConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("OnedriveConfig") - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for OnedriveConfig { type Builder = OnedriveBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/onedrive/config.rs b/core/src/services/onedrive/config.rs new file mode 100644 index 000000000000..40baf69afde2 --- /dev/null +++ b/core/src/services/onedrive/config.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for [OneDrive](https://onedrive.com) backend support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct OnedriveConfig { + /// bearer access token for OneDrive + pub access_token: Option, + /// root path of OneDrive folder. + pub root: Option, +} + +impl Debug for OnedriveConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OnedriveConfig") + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 672fb6c7e1e7..9168394bb766 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -15,13 +15,21 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-onedrive")] mod backend; -mod builder; +#[cfg(feature = "services-onedrive")] mod error; +#[cfg(feature = "services-onedrive")] mod graph_model; +#[cfg(feature = "services-onedrive")] +mod lister; +#[cfg(feature = "services-onedrive")] +mod writer; +#[cfg(feature = "services-onedrive")] +mod builder; +#[cfg(feature = "services-onedrive")] pub use builder::OnedriveBuilder as Onedrive; -pub use builder::OnedriveConfig; -mod lister; -mod writer; +mod config; +pub use config::OnedriveConfig; diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index b9e3d44a7191..aa81ef2e55c3 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -28,8 +28,6 @@ use log::debug; use reqsign::AliyunConfig; use reqsign::AliyunLoader; use reqsign::AliyunOssSigner; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; @@ -37,54 +35,11 @@ use super::lister::OssLister; use super::writer::OssWriter; use crate::raw::*; use crate::services::oss::writer::OssWriters; +use crate::services::OssConfig; use crate::*; const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; -/// Config for Aliyun Object Storage Service (OSS) support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct OssConfig { - /// Root for oss. - pub root: Option, - - /// Endpoint for oss. - pub endpoint: Option, - /// Presign endpoint for oss. - pub presign_endpoint: Option, - /// Bucket for oss. - pub bucket: String, - - // OSS features - /// Server side encryption for oss. - pub server_side_encryption: Option, - /// Server side encryption key id for oss. - pub server_side_encryption_key_id: Option, - /// Allow anonymous for oss. - pub allow_anonymous: bool, - - // authenticate options - /// Access key id for oss. - pub access_key_id: Option, - /// Access key secret for oss. - pub access_key_secret: Option, - /// batch_max_operations - pub batch_max_operations: Option, -} - -impl Debug for OssConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("Builder"); - d.field("root", &self.root) - .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint) - .field("allow_anonymous", &self.allow_anonymous); - - d.finish_non_exhaustive() - } -} - impl Configurator for OssConfig { type Builder = OssBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/oss/config.rs b/core/src/services/oss/config.rs new file mode 100644 index 000000000000..743fbb02ea19 --- /dev/null +++ b/core/src/services/oss/config.rs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Aliyun Object Storage Service (OSS) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct OssConfig { + /// Root for oss. + pub root: Option, + + /// Endpoint for oss. + pub endpoint: Option, + /// Presign endpoint for oss. + pub presign_endpoint: Option, + /// Bucket for oss. + pub bucket: String, + + // OSS features + /// Server side encryption for oss. + pub server_side_encryption: Option, + /// Server side encryption key id for oss. + pub server_side_encryption_key_id: Option, + /// Allow anonymous for oss. + pub allow_anonymous: bool, + + // authenticate options + /// Access key id for oss. + pub access_key_id: Option, + /// Access key secret for oss. + pub access_key_secret: Option, + /// batch_max_operations + pub batch_max_operations: Option, +} + +impl Debug for OssConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("Builder"); + d.field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("allow_anonymous", &self.allow_anonymous); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs index 4839cff2d37f..b5f172e4a70c 100644 --- a/core/src/services/oss/mod.rs +++ b/core/src/services/oss/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::OssBuilder as Oss; -pub use backend::OssConfig; - +#[cfg(feature = "services-oss")] mod core; +#[cfg(feature = "services-oss")] mod error; +#[cfg(feature = "services-oss")] mod lister; +#[cfg(feature = "services-oss")] mod writer; + +#[cfg(feature = "services-oss")] +mod backend; +#[cfg(feature = "services-oss")] +pub use backend::OssBuilder as Oss; + +mod config; +pub use config::OssConfig; diff --git a/core/src/services/pcloud/backend.rs b/core/src/services/pcloud/backend.rs index a07fdd54728b..904b81d848a7 100644 --- a/core/src/services/pcloud/backend.rs +++ b/core/src/services/pcloud/backend.rs @@ -23,8 +23,6 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; @@ -33,37 +31,9 @@ use super::lister::PcloudLister; use super::writer::PcloudWriter; use super::writer::PcloudWriters; use crate::raw::*; +use crate::services::PcloudConfig; use crate::*; -/// Config for Pcloud services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct PcloudConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - ///pCloud endpoint address. - pub endpoint: String, - /// pCloud username. - pub username: Option, - /// pCloud password. - pub password: Option, -} - -impl Debug for PcloudConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Config"); - - ds.field("root", &self.root); - ds.field("endpoint", &self.endpoint); - ds.field("username", &self.username); - - ds.finish() - } -} - impl Configurator for PcloudConfig { type Builder = PcloudBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/pcloud/config.rs b/core/src/services/pcloud/config.rs new file mode 100644 index 000000000000..d09078c24c62 --- /dev/null +++ b/core/src/services/pcloud/config.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Pcloud services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct PcloudConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + ///pCloud endpoint address. + pub endpoint: String, + /// pCloud username. + pub username: Option, + /// pCloud password. + pub password: Option, +} + +impl Debug for PcloudConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Config"); + + ds.field("root", &self.root); + ds.field("endpoint", &self.endpoint); + ds.field("username", &self.username); + + ds.finish() + } +} diff --git a/core/src/services/pcloud/mod.rs b/core/src/services/pcloud/mod.rs index 50fb5e5f53c3..ae3a01c5204b 100644 --- a/core/src/services/pcloud/mod.rs +++ b/core/src/services/pcloud/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::PcloudBuilder as Pcloud; -pub use backend::PcloudConfig; - +#[cfg(feature = "services-pcloud")] mod core; +#[cfg(feature = "services-pcloud")] mod error; +#[cfg(feature = "services-pcloud")] mod lister; +#[cfg(feature = "services-pcloud")] mod writer; + +#[cfg(feature = "services-pcloud")] +mod backend; +#[cfg(feature = "services-pcloud")] +pub use backend::PcloudBuilder as Pcloud; + +mod config; +pub use config::PcloudConfig; diff --git a/core/src/services/persy/backend.rs b/core/src/services/persy/backend.rs index 7b5c0c1fdde1..10b48db08137 100644 --- a/core/src/services/persy/backend.rs +++ b/core/src/services/persy/backend.rs @@ -20,31 +20,17 @@ use std::fmt::Formatter; use std::str; use persy; -use serde::Deserialize; -use serde::Serialize; use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::PersyConfig; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; -/// Config for persy service support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct PersyConfig { - /// That path to the persy data file. The directory in the path must already exist. - pub datafile: Option, - /// That name of the persy segment. - pub segment: Option, - /// That name of the persy index. - pub index: Option, -} - impl Configurator for PersyConfig { type Builder = PersyBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/persy/config.rs b/core/src/services/persy/config.rs new file mode 100644 index 000000000000..d5ef977f3240 --- /dev/null +++ b/core/src/services/persy/config.rs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; + +/// Config for persy service support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct PersyConfig { + /// That path to the persy data file. The directory in the path must already exist. + pub datafile: Option, + /// That name of the persy segment. + pub segment: Option, + /// That name of the persy index. + pub index: Option, +} diff --git a/core/src/services/persy/mod.rs b/core/src/services/persy/mod.rs index a7456fc87be9..005f345ff550 100644 --- a/core/src/services/persy/mod.rs +++ b/core/src/services/persy/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-persy")] mod backend; - +#[cfg(feature = "services-persy")] pub use backend::PersyBuilder as Persy; -pub use backend::PersyConfig; + +mod config; +pub use config::PersyConfig; diff --git a/core/src/services/postgresql/backend.rs b/core/src/services/postgresql/backend.rs index e049eae0d1a7..bc3544d359f1 100644 --- a/core/src/services/postgresql/backend.rs +++ b/core/src/services/postgresql/backend.rs @@ -22,52 +22,14 @@ use std::sync::Arc; use bb8::Pool; use bb8_postgres::PostgresConnectionManager; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use tokio_postgres::Config; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::PostgresqlConfig; use crate::*; -/// Config for PostgreSQL services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct PostgresqlConfig { - /// Root of this backend. - /// - /// All operations will happen under this root. - /// - /// Default to `/` if not set. - pub root: Option, - /// the connection string of postgres server - pub connection_string: Option, - /// the table of postgresql - pub table: Option, - /// the key field of postgresql - pub key_field: Option, - /// the value field of postgresql - pub value_field: Option, -} - -impl Debug for PostgresqlConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("PostgresqlConfig"); - - if self.connection_string.is_some() { - d.field("connection_string", &""); - } - - d.field("root", &self.root) - .field("table", &self.table) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .finish() - } -} - impl Configurator for PostgresqlConfig { type Builder = PostgresqlBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/postgresql/config.rs b/core/src/services/postgresql/config.rs new file mode 100644 index 000000000000..2a3dde90b81e --- /dev/null +++ b/core/src/services/postgresql/config.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for PostgreSQL services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct PostgresqlConfig { + /// Root of this backend. + /// + /// All operations will happen under this root. + /// + /// Default to `/` if not set. + pub root: Option, + /// the connection string of postgres server + pub connection_string: Option, + /// the table of postgresql + pub table: Option, + /// the key field of postgresql + pub key_field: Option, + /// the value field of postgresql + pub value_field: Option, +} + +impl Debug for PostgresqlConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("PostgresqlConfig"); + + if self.connection_string.is_some() { + d.field("connection_string", &""); + } + + d.field("root", &self.root) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .finish() + } +} diff --git a/core/src/services/postgresql/mod.rs b/core/src/services/postgresql/mod.rs index ca185a4ec3b0..1f1d666eb2aa 100644 --- a/core/src/services/postgresql/mod.rs +++ b/core/src/services/postgresql/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-postgresql")] mod backend; +#[cfg(feature = "services-postgresql")] pub use backend::PostgresqlBuilder as Postgresql; -pub use backend::PostgresqlConfig; + +mod config; +pub use config::PostgresqlConfig; diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index e4f3d33276d4..d6dc290e758e 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -19,31 +19,17 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use serde::Deserialize; -use serde::Serialize; use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::RedbConfig; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; -/// Config for redb service support. -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct RedbConfig { - /// path to the redb data directory. - pub datadir: Option, - /// The root for redb. - pub root: Option, - /// The table name for redb. - pub table: Option, -} - impl Configurator for RedbConfig { type Builder = RedbBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/redb/config.rs b/core/src/services/redb/config.rs new file mode 100644 index 000000000000..55b2532f4c3a --- /dev/null +++ b/core/src/services/redb/config.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for redb service support. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct RedbConfig { + /// path to the redb data directory. + pub datadir: Option, + /// The root for redb. + pub root: Option, + /// The table name for redb. + pub table: Option, +} diff --git a/core/src/services/redb/mod.rs b/core/src/services/redb/mod.rs index f75e57fb2f0f..1c44bcd062f7 100644 --- a/core/src/services/redb/mod.rs +++ b/core/src/services/redb/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-redb")] mod backend; - +#[cfg(feature = "services-redb")] pub use backend::RedbBuilder as Redb; -pub use backend::RedbConfig; + +mod config; +pub use config::RedbConfig; diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs index 59e3e09571ae..b0b5c11c235f 100644 --- a/core/src/services/redis/backend.rs +++ b/core/src/services/redis/backend.rs @@ -31,73 +31,16 @@ use redis::ConnectionInfo; use redis::RedisConnectionInfo; use redis::RedisError; use redis::{AsyncCommands, ProtocolVersion}; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::RedisConfig; use crate::*; const DEFAULT_REDIS_ENDPOINT: &str = "tcp://127.0.0.1:6379"; const DEFAULT_REDIS_PORT: u16 = 6379; -/// Config for Redis services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct RedisConfig { - /// network address of the Redis service. Can be "tcp://127.0.0.1:6379", e.g. - /// - /// default is "tcp://127.0.0.1:6379" - pub endpoint: Option, - /// network address of the Redis cluster service. Can be "tcp://127.0.0.1:6379,tcp://127.0.0.1:6380,tcp://127.0.0.1:6381", e.g. - /// - /// default is None - pub cluster_endpoints: Option, - /// the username to connect redis service. - /// - /// default is None - pub username: Option, - /// the password for authentication - /// - /// default is None - pub password: Option, - /// the working directory of the Redis service. Can be "/path/to/dir" - /// - /// default is "/" - pub root: Option, - /// the number of DBs redis can take is unlimited - /// - /// default is db 0 - pub db: i64, - /// The default ttl for put operations. - pub default_ttl: Option, -} - -impl Debug for RedisConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("RedisConfig"); - - d.field("db", &self.db.to_string()); - d.field("root", &self.root); - if let Some(endpoint) = self.endpoint.clone() { - d.field("endpoint", &endpoint); - } - if let Some(cluster_endpoints) = self.cluster_endpoints.clone() { - d.field("cluster_endpoints", &cluster_endpoints); - } - if let Some(username) = self.username.clone() { - d.field("username", &username); - } - if self.password.is_some() { - d.field("password", &""); - } - - d.finish_non_exhaustive() - } -} - impl Configurator for RedisConfig { type Builder = RedisBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/redis/config.rs b/core/src/services/redis/config.rs new file mode 100644 index 000000000000..cfcd8bfb48ea --- /dev/null +++ b/core/src/services/redis/config.rs @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; +use std::time::Duration; + +/// Config for Redis services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct RedisConfig { + /// network address of the Redis service. Can be "tcp://127.0.0.1:6379", e.g. + /// + /// default is "tcp://127.0.0.1:6379" + pub endpoint: Option, + /// network address of the Redis cluster service. Can be "tcp://127.0.0.1:6379,tcp://127.0.0.1:6380,tcp://127.0.0.1:6381", e.g. + /// + /// default is None + pub cluster_endpoints: Option, + /// the username to connect redis service. + /// + /// default is None + pub username: Option, + /// the password for authentication + /// + /// default is None + pub password: Option, + /// the working directory of the Redis service. Can be "/path/to/dir" + /// + /// default is "/" + pub root: Option, + /// the number of DBs redis can take is unlimited + /// + /// default is db 0 + pub db: i64, + /// The default ttl for put operations. + pub default_ttl: Option, +} + +impl Debug for RedisConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("RedisConfig"); + + d.field("db", &self.db.to_string()); + d.field("root", &self.root); + if let Some(endpoint) = self.endpoint.clone() { + d.field("endpoint", &endpoint); + } + if let Some(cluster_endpoints) = self.cluster_endpoints.clone() { + d.field("cluster_endpoints", &cluster_endpoints); + } + if let Some(username) = self.username.clone() { + d.field("username", &username); + } + if self.password.is_some() { + d.field("password", &""); + } + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/redis/mod.rs b/core/src/services/redis/mod.rs index 68668eecfa27..a1dc12d620f8 100644 --- a/core/src/services/redis/mod.rs +++ b/core/src/services/redis/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-redis")] mod backend; +#[cfg(feature = "services-redis")] pub use backend::RedisBuilder as Redis; -pub use backend::RedisConfig; + +mod config; +pub use config::RedisConfig; diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs index e117b41aaee0..9a139f03d1cc 100644 --- a/core/src/services/rocksdb/backend.rs +++ b/core/src/services/rocksdb/backend.rs @@ -20,28 +20,14 @@ use std::fmt::Formatter; use std::sync::Arc; use rocksdb::DB; -use serde::Deserialize; -use serde::Serialize; use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::RocksdbConfig; use crate::Result; use crate::*; -/// Config for Rocksdb Service. -#[derive(Default, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct RocksdbConfig { - /// The path to the rocksdb data directory. - pub datadir: Option, - /// the working directory of the service. Can be "/path/to/dir" - /// - /// default is "/" - pub root: Option, -} - impl Configurator for RocksdbConfig { type Builder = RocksdbBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/rocksdb/config.rs b/core/src/services/rocksdb/config.rs new file mode 100644 index 000000000000..29dca941b161 --- /dev/null +++ b/core/src/services/rocksdb/config.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Config for Rocksdb Service. +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct RocksdbConfig { + /// The path to the rocksdb data directory. + pub datadir: Option, + /// the working directory of the service. Can be "/path/to/dir" + /// + /// default is "/" + pub root: Option, +} diff --git a/core/src/services/rocksdb/mod.rs b/core/src/services/rocksdb/mod.rs index 14ed70ac3dea..d859ce4c7ddb 100644 --- a/core/src/services/rocksdb/mod.rs +++ b/core/src/services/rocksdb/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-rocksdb")] mod backend; +#[cfg(feature = "services-rocksdb")] pub use backend::RocksdbBuilder as Rocksdb; -pub use backend::RocksdbConfig; + +mod config; +pub use config::RocksdbConfig; diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index f124f0cea343..9206d41fd2c3 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -39,8 +39,6 @@ use reqsign::AwsCredentialLoad; use reqsign::AwsDefaultLoader; use reqsign::AwsV4Signer; use reqwest::Url; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; @@ -49,6 +47,7 @@ use super::lister::S3Lister; use super::writer::S3Writer; use super::writer::S3Writers; use crate::raw::*; +use crate::services::S3Config; use crate::*; /// Allow constructing correct region endpoint if user gives a global endpoint. @@ -64,169 +63,6 @@ static ENDPOINT_TEMPLATES: Lazy> = Lazy::new const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000; -/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct S3Config { - /// root of this backend. - /// - /// All operations will happen under this root. - /// - /// default to `/` if not set. - pub root: Option, - /// bucket name of this backend. - /// - /// required. - pub bucket: String, - /// endpoint of this backend. - /// - /// Endpoint must be full uri, e.g. - /// - /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` - /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` - /// - Aliyun OSS: `https://{region}.aliyuncs.com` - /// - Tencent COS: `https://cos.{region}.myqcloud.com` - /// - Minio: `http://127.0.0.1:9000` - /// - /// If user inputs endpoint without scheme like "s3.amazonaws.com", we - /// will prepend "https://" before it. - /// - /// default to `https://s3.amazonaws.com` if not set. - pub endpoint: Option, - /// Region represent the signing region of this endpoint. This is required - /// if you are using the default AWS S3 endpoint. - /// - /// If using a custom endpoint, - /// - If region is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub region: Option, - - /// access_key_id of this backend. - /// - /// - If access_key_id is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub access_key_id: Option, - /// secret_access_key of this backend. - /// - /// - If secret_access_key is set, we will take user's input first. - /// - If not, we will try to load it from environment. - pub secret_access_key: Option, - /// session_token (aka, security token) of this backend. - /// - /// This token will expire after sometime, it's recommended to set session_token - /// by hand. - pub session_token: Option, - /// role_arn for this backend. - /// - /// If `role_arn` is set, we will use already known config as source - /// credential to assume role with `role_arn`. - pub role_arn: Option, - /// external_id for this backend. - pub external_id: Option, - /// role_session_name for this backend. - pub role_session_name: Option, - /// Disable config load so that opendal will not load config from - /// environment. - /// - /// For examples: - /// - /// - envs like `AWS_ACCESS_KEY_ID` - /// - files like `~/.aws/config` - pub disable_config_load: bool, - /// Disable load credential from ec2 metadata. - /// - /// This option is used to disable the default behavior of opendal - /// to load credential from ec2 metadata, a.k.a, IMDSv2 - pub disable_ec2_metadata: bool, - /// Allow anonymous will allow opendal to send request without signing - /// when credential is not loaded. - pub allow_anonymous: bool, - /// server_side_encryption for this backend. - /// - /// Available values: `AES256`, `aws:kms`. - pub server_side_encryption: Option, - /// server_side_encryption_aws_kms_key_id for this backend - /// - /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` - /// is not set, S3 will use aws managed kms key to encrypt data. - /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` - /// is a valid kms key id, S3 will use the provided kms key to encrypt data. - /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be - /// returned. - /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` - /// is a noop. - pub server_side_encryption_aws_kms_key_id: Option, - /// server_side_encryption_customer_algorithm for this backend. - /// - /// Available values: `AES256`. - pub server_side_encryption_customer_algorithm: Option, - /// server_side_encryption_customer_key for this backend. - /// - /// # Value - /// - /// base64 encoded key that matches algorithm specified in - /// `server_side_encryption_customer_algorithm`. - pub server_side_encryption_customer_key: Option, - /// Set server_side_encryption_customer_key_md5 for this backend. - /// - /// # Value - /// - /// MD5 digest of key specified in `server_side_encryption_customer_key`. - pub server_side_encryption_customer_key_md5: Option, - /// default storage_class for this backend. - /// - /// Available values: - /// - `DEEP_ARCHIVE` - /// - `GLACIER` - /// - `GLACIER_IR` - /// - `INTELLIGENT_TIERING` - /// - `ONEZONE_IA` - /// - `OUTPOSTS` - /// - `REDUCED_REDUNDANCY` - /// - `STANDARD` - /// - `STANDARD_IA` - /// - /// S3 compatible services don't support all of them - pub default_storage_class: Option, - /// Enable virtual host style so that opendal will send API requests - /// in virtual host style instead of path style. - /// - /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` - /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` - pub enable_virtual_host_style: bool, - /// Set maximum batch operations of this backend. - /// - /// Some compatible services have a limit on the number of operations in a batch request. - /// For example, R2 could return `Internal Error` while batch delete 1000 files. - /// - /// Please tune this value based on services' document. - pub batch_max_operations: Option, - /// Disable stat with override so that opendal will not send stat request with override queries. - /// - /// For example, R2 doesn't support stat with `response_content_type` query. - pub disable_stat_with_override: bool, - /// Checksum Algorithm to use when sending checksums in HTTP headers. - /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example. - /// - /// Available options: - /// - "crc32c" - pub checksum_algorithm: Option, -} - -impl Debug for S3Config { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("S3Config"); - - d.field("root", &self.root) - .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint) - .field("region", &self.region); - - d.finish_non_exhaustive() - } -} - impl Configurator for S3Config { type Builder = S3Builder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/s3/config.rs b/core/src/services/s3/config.rs new file mode 100644 index 000000000000..2cee50ea391d --- /dev/null +++ b/core/src/services/s3/config.rs @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct S3Config { + /// root of this backend. + /// + /// All operations will happen under this root. + /// + /// default to `/` if not set. + pub root: Option, + /// bucket name of this backend. + /// + /// required. + pub bucket: String, + /// endpoint of this backend. + /// + /// Endpoint must be full uri, e.g. + /// + /// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com` + /// - Cloudflare R2: `https://.r2.cloudflarestorage.com` + /// - Aliyun OSS: `https://{region}.aliyuncs.com` + /// - Tencent COS: `https://cos.{region}.myqcloud.com` + /// - Minio: `http://127.0.0.1:9000` + /// + /// If user inputs endpoint without scheme like "s3.amazonaws.com", we + /// will prepend "https://" before it. + /// + /// default to `https://s3.amazonaws.com` if not set. + pub endpoint: Option, + /// Region represent the signing region of this endpoint. This is required + /// if you are using the default AWS S3 endpoint. + /// + /// If using a custom endpoint, + /// - If region is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub region: Option, + + /// access_key_id of this backend. + /// + /// - If access_key_id is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub access_key_id: Option, + /// secret_access_key of this backend. + /// + /// - If secret_access_key is set, we will take user's input first. + /// - If not, we will try to load it from environment. + pub secret_access_key: Option, + /// session_token (aka, security token) of this backend. + /// + /// This token will expire after sometime, it's recommended to set session_token + /// by hand. + pub session_token: Option, + /// role_arn for this backend. + /// + /// If `role_arn` is set, we will use already known config as source + /// credential to assume role with `role_arn`. + pub role_arn: Option, + /// external_id for this backend. + pub external_id: Option, + /// role_session_name for this backend. + pub role_session_name: Option, + /// Disable config load so that opendal will not load config from + /// environment. + /// + /// For examples: + /// + /// - envs like `AWS_ACCESS_KEY_ID` + /// - files like `~/.aws/config` + pub disable_config_load: bool, + /// Disable load credential from ec2 metadata. + /// + /// This option is used to disable the default behavior of opendal + /// to load credential from ec2 metadata, a.k.a, IMDSv2 + pub disable_ec2_metadata: bool, + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + pub allow_anonymous: bool, + /// server_side_encryption for this backend. + /// + /// Available values: `AES256`, `aws:kms`. + pub server_side_encryption: Option, + /// server_side_encryption_aws_kms_key_id for this backend + /// + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is not set, S3 will use aws managed kms key to encrypt data. + /// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id` + /// is a valid kms key id, S3 will use the provided kms key to encrypt data. + /// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be + /// returned. + /// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id` + /// is a noop. + pub server_side_encryption_aws_kms_key_id: Option, + /// server_side_encryption_customer_algorithm for this backend. + /// + /// Available values: `AES256`. + pub server_side_encryption_customer_algorithm: Option, + /// server_side_encryption_customer_key for this backend. + /// + /// # Value + /// + /// base64 encoded key that matches algorithm specified in + /// `server_side_encryption_customer_algorithm`. + pub server_side_encryption_customer_key: Option, + /// Set server_side_encryption_customer_key_md5 for this backend. + /// + /// # Value + /// + /// MD5 digest of key specified in `server_side_encryption_customer_key`. + pub server_side_encryption_customer_key_md5: Option, + /// default storage_class for this backend. + /// + /// Available values: + /// - `DEEP_ARCHIVE` + /// - `GLACIER` + /// - `GLACIER_IR` + /// - `INTELLIGENT_TIERING` + /// - `ONEZONE_IA` + /// - `OUTPOSTS` + /// - `REDUCED_REDUNDANCY` + /// - `STANDARD` + /// - `STANDARD_IA` + /// + /// S3 compatible services don't support all of them + pub default_storage_class: Option, + /// Enable virtual host style so that opendal will send API requests + /// in virtual host style instead of path style. + /// + /// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name` + /// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` + pub enable_virtual_host_style: bool, + /// Set maximum batch operations of this backend. + /// + /// Some compatible services have a limit on the number of operations in a batch request. + /// For example, R2 could return `Internal Error` while batch delete 1000 files. + /// + /// Please tune this value based on services' document. + pub batch_max_operations: Option, + /// Disable stat with override so that opendal will not send stat request with override queries. + /// + /// For example, R2 doesn't support stat with `response_content_type` query. + pub disable_stat_with_override: bool, + /// Checksum Algorithm to use when sending checksums in HTTP headers. + /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example. + /// + /// Available options: + /// - "crc32c" + pub checksum_algorithm: Option, +} + +impl Debug for S3Config { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("S3Config"); + + d.field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("region", &self.region); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/s3/mod.rs b/core/src/services/s3/mod.rs index 4e2f283cb215..b4161a9d66b2 100644 --- a/core/src/services/s3/mod.rs +++ b/core/src/services/s3/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::S3Builder as S3; -pub use backend::S3Config; - +#[cfg(feature = "services-s3")] mod core; +#[cfg(feature = "services-s3")] mod error; +#[cfg(feature = "services-s3")] mod lister; +#[cfg(feature = "services-s3")] mod writer; + +#[cfg(feature = "services-s3")] +mod backend; +#[cfg(feature = "services-s3")] +pub use backend::S3Builder as S3; + +mod config; +pub use config::S3Config; diff --git a/core/src/services/seafile/backend.rs b/core/src/services/seafile/backend.rs index 10b71a1fbe76..29029b6b23bd 100644 --- a/core/src/services/seafile/backend.rs +++ b/core/src/services/seafile/backend.rs @@ -22,8 +22,6 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use tokio::sync::RwLock; use super::core::parse_dir_detail; @@ -35,42 +33,9 @@ use super::lister::SeafileLister; use super::writer::SeafileWriter; use super::writer::SeafileWriters; use crate::raw::*; +use crate::services::SeafileConfig; use crate::*; -/// Config for seafile services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SeafileConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// endpoint address of this backend. - pub endpoint: Option, - /// username of this backend. - pub username: Option, - /// password of this backend. - pub password: Option, - /// repo_name of this backend. - /// - /// required. - pub repo_name: String, -} - -impl Debug for SeafileConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("SeafileConfig"); - - d.field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("username", &self.username) - .field("repo_name", &self.repo_name); - - d.finish_non_exhaustive() - } -} - impl Configurator for SeafileConfig { type Builder = SeafileBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/seafile/config.rs b/core/src/services/seafile/config.rs new file mode 100644 index 000000000000..7be10fb91e8e --- /dev/null +++ b/core/src/services/seafile/config.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for seafile services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SeafileConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// endpoint address of this backend. + pub endpoint: Option, + /// username of this backend. + pub username: Option, + /// password of this backend. + pub password: Option, + /// repo_name of this backend. + /// + /// required. + pub repo_name: String, +} + +impl Debug for SeafileConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("SeafileConfig"); + + d.field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("username", &self.username) + .field("repo_name", &self.repo_name); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/seafile/mod.rs b/core/src/services/seafile/mod.rs index dc9e1ccce078..4ba76b05b7d5 100644 --- a/core/src/services/seafile/mod.rs +++ b/core/src/services/seafile/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::SeafileBuilder as Seafile; -pub use backend::SeafileConfig; - +#[cfg(feature = "services-seafile")] mod core; +#[cfg(feature = "services-seafile")] mod error; +#[cfg(feature = "services-seafile")] mod lister; +#[cfg(feature = "services-seafile")] mod writer; + +#[cfg(feature = "services-seafile")] +mod backend; +#[cfg(feature = "services-seafile")] +pub use backend::SeafileBuilder as Seafile; + +mod config; +pub use config::SeafileConfig; diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 637164e68964..0d915572abfb 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -29,8 +29,6 @@ use openssh::KnownHosts; use openssh::SessionBuilder; use openssh_sftp_client::Sftp; use openssh_sftp_client::SftpOptions; -use serde::Deserialize; -use serde::Serialize; use tokio::io::AsyncSeekExt; use tokio::sync::OnceCell; @@ -42,36 +40,9 @@ use super::lister::SftpLister; use super::reader::SftpReader; use super::writer::SftpWriter; use crate::raw::*; +use crate::services::SftpConfig; use crate::*; -/// Config for Sftp Service support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SftpConfig { - /// endpoint of this backend - pub endpoint: Option, - /// root of this backend - pub root: Option, - /// user of this backend - pub user: Option, - /// key of this backend - pub key: Option, - /// known_hosts_strategy of this backend - pub known_hosts_strategy: Option, - /// enable_copy of this backend - pub enable_copy: bool, -} - -impl Debug for SftpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SftpConfig") - .field("endpoint", &self.endpoint) - .field("root", &self.root) - .finish_non_exhaustive() - } -} - impl Configurator for SftpConfig { type Builder = SftpBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/sftp/config.rs b/core/src/services/sftp/config.rs new file mode 100644 index 000000000000..b3187bf3eaf1 --- /dev/null +++ b/core/src/services/sftp/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Sftp Service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SftpConfig { + /// endpoint of this backend + pub endpoint: Option, + /// root of this backend + pub root: Option, + /// user of this backend + pub user: Option, + /// key of this backend + pub key: Option, + /// known_hosts_strategy of this backend + pub known_hosts_strategy: Option, + /// enable_copy of this backend + pub enable_copy: bool, +} + +impl Debug for SftpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SftpConfig") + .field("endpoint", &self.endpoint) + .field("root", &self.root) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/sftp/mod.rs b/core/src/services/sftp/mod.rs index 001898171b0f..b17c0e0007bb 100644 --- a/core/src/services/sftp/mod.rs +++ b/core/src/services/sftp/mod.rs @@ -15,12 +15,21 @@ // specific language governing permissions and limitations // under the License. -pub use backend::SftpBuilder as Sftp; -pub use backend::SftpConfig; - -mod backend; +#[cfg(feature = "services-sftp")] mod error; +#[cfg(feature = "services-sftp")] mod lister; +#[cfg(feature = "services-sftp")] mod reader; +#[cfg(feature = "services-sftp")] mod utils; +#[cfg(feature = "services-sftp")] mod writer; + +#[cfg(feature = "services-sftp")] +mod backend; +#[cfg(feature = "services-sftp")] +pub use backend::SftpBuilder as Sftp; + +mod config; +pub use config::SftpConfig; diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs index b100c4e6877b..bb3a81f441a6 100644 --- a/core/src/services/sled/backend.rs +++ b/core/src/services/sled/backend.rs @@ -19,12 +19,11 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::str; -use serde::Deserialize; -use serde::Serialize; use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::SledConfig; use crate::Builder; use crate::Error; use crate::ErrorKind; @@ -34,29 +33,6 @@ use crate::*; // https://github.com/spacejam/sled/blob/69294e59c718289ab3cb6bd03ac3b9e1e072a1e7/src/db.rs#L5 const DEFAULT_TREE_ID: &str = r#"__sled__default"#; -/// Config for Sled services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SledConfig { - /// That path to the sled data directory. - pub datadir: Option, - /// The root for sled. - pub root: Option, - /// The tree for sled. - pub tree: Option, -} - -impl Debug for SledConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SledConfig") - .field("datadir", &self.datadir) - .field("root", &self.root) - .field("tree", &self.tree) - .finish() - } -} - impl Configurator for SledConfig { type Builder = SledBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/sled/config.rs b/core/src/services/sled/config.rs new file mode 100644 index 000000000000..d5e1f73e340e --- /dev/null +++ b/core/src/services/sled/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Sled services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SledConfig { + /// That path to the sled data directory. + pub datadir: Option, + /// The root for sled. + pub root: Option, + /// The tree for sled. + pub tree: Option, +} + +impl Debug for SledConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SledConfig") + .field("datadir", &self.datadir) + .field("root", &self.root) + .field("tree", &self.tree) + .finish() + } +} diff --git a/core/src/services/sled/mod.rs b/core/src/services/sled/mod.rs index af28a3e9b2d0..f8771b65f39c 100644 --- a/core/src/services/sled/mod.rs +++ b/core/src/services/sled/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-sled")] mod backend; - +#[cfg(feature = "services-sled")] pub use backend::SledBuilder as Sled; -pub use backend::SledConfig; + +mod config; +pub use config::SledConfig; diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index 4e9538509cb4..de5b77aed60e 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -20,62 +20,13 @@ use std::fmt::Formatter; use rusqlite::params; use rusqlite::Connection; -use serde::Deserialize; -use serde::Serialize; use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; +use crate::services::SqliteConfig; use crate::*; -/// Config for Sqlite support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SqliteConfig { - /// Set the connection_string of the sqlite service. - /// - /// This connection string is used to connect to the sqlite service. There are url based formats: - /// - /// ## Url - /// - /// This format resembles the url format of the sqlite client. The format is: `file://[path]?flag` - /// - /// - `file://data.db` - /// - /// For more information, please refer to [Opening A New Database Connection](http://www.sqlite.org/c3ref/open.html) - pub connection_string: Option, - - /// Set the table name of the sqlite service to read/write. - pub table: Option, - /// Set the key field name of the sqlite service to read/write. - /// - /// Default to `key` if not specified. - pub key_field: Option, - /// Set the value field name of the sqlite service to read/write. - /// - /// Default to `value` if not specified. - pub value_field: Option, - /// set the working directory, all operations will be performed under it. - /// - /// default: "/" - pub root: Option, -} - -impl Debug for SqliteConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("SqliteConfig"); - - d.field("connection_string", &self.connection_string) - .field("table", &self.table) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .field("root", &self.root); - - d.finish_non_exhaustive() - } -} - impl Configurator for SqliteConfig { type Builder = SqliteBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/sqlite/config.rs b/core/src/services/sqlite/config.rs new file mode 100644 index 000000000000..55b2fd35eb40 --- /dev/null +++ b/core/src/services/sqlite/config.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Sqlite support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SqliteConfig { + /// Set the connection_string of the sqlite service. + /// + /// This connection string is used to connect to the sqlite service. There are url based formats: + /// + /// ## Url + /// + /// This format resembles the url format of the sqlite client. The format is: `file://[path]?flag` + /// + /// - `file://data.db` + /// + /// For more information, please refer to [Opening A New Database Connection](http://www.sqlite.org/c3ref/open.html) + pub connection_string: Option, + + /// Set the table name of the sqlite service to read/write. + pub table: Option, + /// Set the key field name of the sqlite service to read/write. + /// + /// Default to `key` if not specified. + pub key_field: Option, + /// Set the value field name of the sqlite service to read/write. + /// + /// Default to `value` if not specified. + pub value_field: Option, + /// set the working directory, all operations will be performed under it. + /// + /// default: "/" + pub root: Option, +} + +impl Debug for SqliteConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("SqliteConfig"); + + d.field("connection_string", &self.connection_string) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .field("root", &self.root); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/sqlite/mod.rs b/core/src/services/sqlite/mod.rs index d18accf80018..32dfb9177f53 100644 --- a/core/src/services/sqlite/mod.rs +++ b/core/src/services/sqlite/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-sqlite")] mod backend; +#[cfg(feature = "services-sqlite")] pub use backend::SqliteBuilder as Sqlite; -pub use backend::SqliteConfig; + +mod config; +pub use config::SqliteConfig; diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs index 773ffb381a5f..cc6897fdfae2 100644 --- a/core/src/services/supabase/backend.rs +++ b/core/src/services/supabase/backend.rs @@ -22,39 +22,14 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; use super::writer::*; use crate::raw::*; +use crate::services::SupabaseConfig; use crate::*; -/// Config for supabase service support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SupabaseConfig { - root: Option, - bucket: String, - endpoint: Option, - key: Option, - // TODO(1) optional public, currently true always - // TODO(2) optional file_size_limit, currently 0 - // TODO(3) optional allowed_mime_types, currently only string -} - -impl Debug for SupabaseConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SupabaseConfig") - .field("root", &self.root) - .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint) - .finish_non_exhaustive() - } -} - impl Configurator for SupabaseConfig { type Builder = SupabaseBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/supabase/config.rs b/core/src/services/supabase/config.rs new file mode 100644 index 000000000000..d5d2d671efb8 --- /dev/null +++ b/core/src/services/supabase/config.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for supabase service support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SupabaseConfig { + /// The root for supabase service. + pub root: Option, + /// The bucket for supabase service. + pub bucket: String, + /// The endpoint for supabase service. + pub endpoint: Option, + /// The key for supabase service. + pub key: Option, + // TODO(1) optional public, currently true always + // TODO(2) optional file_size_limit, currently 0 + // TODO(3) optional allowed_mime_types, currently only string +} + +impl Debug for SupabaseConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SupabaseConfig") + .field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/supabase/mod.rs b/core/src/services/supabase/mod.rs index af59bfcc57f6..4b485a2f10c4 100644 --- a/core/src/services/supabase/mod.rs +++ b/core/src/services/supabase/mod.rs @@ -15,9 +15,17 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::SupabaseBuilder as Supabase; -pub use backend::SupabaseConfig; +#[cfg(feature = "services-supabase")] mod core; +#[cfg(feature = "services-supabase")] mod error; +#[cfg(feature = "services-supabase")] mod writer; + +#[cfg(feature = "services-supabase")] +mod backend; +#[cfg(feature = "services-supabase")] +pub use backend::SupabaseBuilder as Supabase; + +mod config; +pub use config::SupabaseConfig; diff --git a/core/src/services/surrealdb/backend.rs b/core/src/services/surrealdb/backend.rs index b1129b40366e..4ddbd3ffb7b7 100644 --- a/core/src/services/surrealdb/backend.rs +++ b/core/src/services/surrealdb/backend.rs @@ -19,8 +19,6 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use serde::Deserialize; -use serde::Serialize; use surrealdb::engine::any::Any; use surrealdb::opt::auth::Database; use surrealdb::Surreal; @@ -29,50 +27,9 @@ use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::normalize_root; use crate::raw::Access; +use crate::services::SurrealdbConfig; use crate::*; -/// Config for Surrealdb services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SurrealdbConfig { - /// The connection string for surrealdb. - pub connection_string: Option, - /// The username for surrealdb. - pub username: Option, - /// The password for surrealdb. - pub password: Option, - /// The namespace for surrealdb. - pub namespace: Option, - /// The database for surrealdb. - pub database: Option, - /// The table for surrealdb. - pub table: Option, - /// The key field for surrealdb. - pub key_field: Option, - /// The value field for surrealdb. - pub value_field: Option, - /// The root for surrealdb. - pub root: Option, -} - -impl Debug for SurrealdbConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("SurrealdbConfig"); - - d.field("connection_string", &self.connection_string) - .field("username", &self.username) - .field("password", &"") - .field("namespace", &self.namespace) - .field("database", &self.database) - .field("table", &self.table) - .field("key_field", &self.key_field) - .field("value_field", &self.value_field) - .field("root", &self.root) - .finish() - } -} - impl Configurator for SurrealdbConfig { type Builder = SurrealdbBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/surrealdb/config.rs b/core/src/services/surrealdb/config.rs new file mode 100644 index 000000000000..cb1c48f2ff2b --- /dev/null +++ b/core/src/services/surrealdb/config.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Surrealdb services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SurrealdbConfig { + /// The connection string for surrealdb. + pub connection_string: Option, + /// The username for surrealdb. + pub username: Option, + /// The password for surrealdb. + pub password: Option, + /// The namespace for surrealdb. + pub namespace: Option, + /// The database for surrealdb. + pub database: Option, + /// The table for surrealdb. + pub table: Option, + /// The key field for surrealdb. + pub key_field: Option, + /// The value field for surrealdb. + pub value_field: Option, + /// The root for surrealdb. + pub root: Option, +} + +impl Debug for SurrealdbConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("SurrealdbConfig"); + + d.field("connection_string", &self.connection_string) + .field("username", &self.username) + .field("password", &"") + .field("namespace", &self.namespace) + .field("database", &self.database) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .field("root", &self.root) + .finish() + } +} diff --git a/core/src/services/surrealdb/mod.rs b/core/src/services/surrealdb/mod.rs index 0f5d19b38f0b..40cabaf71d9d 100644 --- a/core/src/services/surrealdb/mod.rs +++ b/core/src/services/surrealdb/mod.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-surrealdb")] mod backend; +#[cfg(feature = "services-surrealdb")] pub use backend::SurrealdbBuilder as Surrealdb; -pub use backend::SurrealdbConfig; + +mod config; +pub use config::SurrealdbConfig; diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs index 89a1f83d1993..3056567bf0a5 100644 --- a/core/src/services/swift/backend.rs +++ b/core/src/services/swift/backend.rs @@ -22,47 +22,15 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; use super::lister::SwiftLister; use super::writer::SwiftWriter; use crate::raw::*; +use crate::services::SwiftConfig; use crate::*; -/// Config for OpenStack Swift support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct SwiftConfig { - /// The endpoint for Swift. - pub endpoint: Option, - /// The container for Swift. - pub container: Option, - /// The root for Swift. - pub root: Option, - /// The token for Swift. - pub token: Option, -} - -impl Debug for SwiftConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("SwiftConfig"); - - ds.field("root", &self.root); - ds.field("endpoint", &self.endpoint); - ds.field("container", &self.container); - - if self.token.is_some() { - ds.field("token", &""); - } - - ds.finish() - } -} - impl Configurator for SwiftConfig { type Builder = SwiftBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/swift/config.rs b/core/src/services/swift/config.rs new file mode 100644 index 000000000000..62ad1d64a5a6 --- /dev/null +++ b/core/src/services/swift/config.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for OpenStack Swift support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct SwiftConfig { + /// The endpoint for Swift. + pub endpoint: Option, + /// The container for Swift. + pub container: Option, + /// The root for Swift. + pub root: Option, + /// The token for Swift. + pub token: Option, +} + +impl Debug for SwiftConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("SwiftConfig"); + + ds.field("root", &self.root); + ds.field("endpoint", &self.endpoint); + ds.field("container", &self.container); + + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} diff --git a/core/src/services/swift/mod.rs b/core/src/services/swift/mod.rs index e15b8efd02ed..451ca66231e6 100644 --- a/core/src/services/swift/mod.rs +++ b/core/src/services/swift/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::SwiftBuilder as Swift; -pub use backend::SwiftConfig; - +#[cfg(feature = "services-swift")] mod core; +#[cfg(feature = "services-swift")] mod error; +#[cfg(feature = "services-swift")] mod lister; +#[cfg(feature = "services-swift")] mod writer; + +#[cfg(feature = "services-swift")] +mod backend; +#[cfg(feature = "services-swift")] +pub use backend::SwiftBuilder as Swift; + +mod config; +pub use config::SwiftConfig; diff --git a/core/src/services/tikv/backend.rs b/core/src/services/tikv/backend.rs index cb113e956ee4..6f23f156ee70 100644 --- a/core/src/services/tikv/backend.rs +++ b/core/src/services/tikv/backend.rs @@ -18,14 +18,13 @@ use std::fmt::Debug; use std::fmt::Formatter; -use serde::Deserialize; -use serde::Serialize; use tikv_client::Config; use tikv_client::RawClient; use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::Access; +use crate::services::TikvConfig; use crate::Builder; use crate::Capability; use crate::Error; @@ -33,36 +32,6 @@ use crate::ErrorKind; use crate::Scheme; use crate::*; -/// Config for Tikv services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct TikvConfig { - /// network address of the TiKV service. - pub endpoints: Option>, - /// whether using insecure connection to TiKV - pub insecure: bool, - /// certificate authority file path - pub ca_path: Option, - /// cert path - pub cert_path: Option, - /// key path - pub key_path: Option, -} - -impl Debug for TikvConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("TikvConfig"); - - d.field("endpoints", &self.endpoints) - .field("insecure", &self.insecure) - .field("ca_path", &self.ca_path) - .field("cert_path", &self.cert_path) - .field("key_path", &self.key_path) - .finish() - } -} - impl Configurator for TikvConfig { type Builder = TikvBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/tikv/config.rs b/core/src/services/tikv/config.rs new file mode 100644 index 000000000000..81a5eb5e3a15 --- /dev/null +++ b/core/src/services/tikv/config.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Tikv services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct TikvConfig { + /// network address of the TiKV service. + pub endpoints: Option>, + /// whether using insecure connection to TiKV + pub insecure: bool, + /// certificate authority file path + pub ca_path: Option, + /// cert path + pub cert_path: Option, + /// key path + pub key_path: Option, +} + +impl Debug for TikvConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("TikvConfig"); + + d.field("endpoints", &self.endpoints) + .field("insecure", &self.insecure) + .field("ca_path", &self.ca_path) + .field("cert_path", &self.cert_path) + .field("key_path", &self.key_path) + .finish() + } +} diff --git a/core/src/services/tikv/mod.rs b/core/src/services/tikv/mod.rs index ae9c15bd68c1..36bf76344403 100644 --- a/core/src/services/tikv/mod.rs +++ b/core/src/services/tikv/mod.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-tikv")] mod backend; - +#[cfg(feature = "services-tikv")] pub use backend::TikvBuilder as Tikv; -pub use backend::TikvConfig; + +mod config; +pub use config::TikvConfig; diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs index 1a9c4e03575b..ed96d1092f6e 100644 --- a/core/src/services/upyun/backend.rs +++ b/core/src/services/upyun/backend.rs @@ -22,8 +22,6 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; @@ -31,37 +29,9 @@ use super::lister::UpyunLister; use super::writer::UpyunWriter; use super::writer::UpyunWriters; use crate::raw::*; +use crate::services::UpyunConfig; use crate::*; -/// Config for upyun services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct UpyunConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// bucket address of this backend. - pub bucket: String, - /// username of this backend. - pub operator: Option, - /// password of this backend. - pub password: Option, -} - -impl Debug for UpyunConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Config"); - - ds.field("root", &self.root); - ds.field("bucket", &self.bucket); - ds.field("operator", &self.operator); - - ds.finish() - } -} - impl Configurator for UpyunConfig { type Builder = UpyunBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/upyun/config.rs b/core/src/services/upyun/config.rs new file mode 100644 index 000000000000..5d45af6c349f --- /dev/null +++ b/core/src/services/upyun/config.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for upyun services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct UpyunConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// bucket address of this backend. + pub bucket: String, + /// username of this backend. + pub operator: Option, + /// password of this backend. + pub password: Option, +} + +impl Debug for UpyunConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Config"); + + ds.field("root", &self.root); + ds.field("bucket", &self.bucket); + ds.field("operator", &self.operator); + + ds.finish() + } +} diff --git a/core/src/services/upyun/mod.rs b/core/src/services/upyun/mod.rs index 039f2aa22944..ca78feea823d 100644 --- a/core/src/services/upyun/mod.rs +++ b/core/src/services/upyun/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::UpyunBuilder as Upyun; -pub use backend::UpyunConfig; - +#[cfg(feature = "services-upyun")] mod core; +#[cfg(feature = "services-upyun")] mod error; +#[cfg(feature = "services-upyun")] mod lister; +#[cfg(feature = "services-upyun")] mod writer; + +#[cfg(feature = "services-upyun")] +mod backend; +#[cfg(feature = "services-upyun")] +pub use backend::UpyunBuilder as Upyun; + +mod config; +pub use config::UpyunConfig; diff --git a/core/src/services/vercel_artifacts/builder.rs b/core/src/services/vercel_artifacts/builder.rs index 8618e49ed268..8ccd9591e77f 100644 --- a/core/src/services/vercel_artifacts/builder.rs +++ b/core/src/services/vercel_artifacts/builder.rs @@ -18,32 +18,13 @@ use std::fmt::Debug; use std::fmt::Formatter; -use serde::Deserialize; -use serde::Serialize; - use super::backend::VercelArtifactsBackend; use crate::raw::Access; use crate::raw::HttpClient; +use crate::services::VercelArtifactsConfig; use crate::Scheme; use crate::*; -/// Config for Vercel Cache support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct VercelArtifactsConfig { - /// The access token for Vercel. - pub access_token: Option, -} - -impl Debug for VercelArtifactsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VercelArtifactsConfig") - .field("access_token", &"") - .finish() - } -} - impl Configurator for VercelArtifactsConfig { type Builder = VercelArtifactsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/vercel_artifacts/config.rs b/core/src/services/vercel_artifacts/config.rs new file mode 100644 index 000000000000..863cdf175c7e --- /dev/null +++ b/core/src/services/vercel_artifacts/config.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for Vercel Cache support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct VercelArtifactsConfig { + /// The access token for Vercel. + pub access_token: Option, +} + +impl Debug for VercelArtifactsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VercelArtifactsConfig") + .field("access_token", &"") + .finish() + } +} diff --git a/core/src/services/vercel_artifacts/mod.rs b/core/src/services/vercel_artifacts/mod.rs index eac0f6e43a7f..2dd496cdb3dd 100644 --- a/core/src/services/vercel_artifacts/mod.rs +++ b/core/src/services/vercel_artifacts/mod.rs @@ -15,10 +15,17 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "services-vercel-artifacts")] mod backend; -mod builder; +#[cfg(feature = "services-vercel-artifacts")] mod error; +#[cfg(feature = "services-vercel-artifacts")] mod writer; +#[cfg(feature = "services-vercel-artifacts")] +mod builder; +#[cfg(feature = "services-vercel-artifacts")] pub use builder::VercelArtifactsBuilder as VercelArtifacts; -pub use builder::VercelArtifactsConfig; + +mod config; +pub use config::VercelArtifactsConfig; diff --git a/core/src/services/vercel_blob/backend.rs b/core/src/services/vercel_blob/backend.rs index fb308b516f90..35fdfdbcd1a5 100644 --- a/core/src/services/vercel_blob/backend.rs +++ b/core/src/services/vercel_blob/backend.rs @@ -23,8 +23,6 @@ use bytes::Buf; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::parse_blob; use super::core::Blob; @@ -34,31 +32,9 @@ use super::lister::VercelBlobLister; use super::writer::VercelBlobWriter; use super::writer::VercelBlobWriters; use crate::raw::*; +use crate::services::VercelBlobConfig; use crate::*; -/// Config for VercelBlob services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct VercelBlobConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// vercel blob token. - pub token: String, -} - -impl Debug for VercelBlobConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Config"); - - ds.field("root", &self.root); - - ds.finish() - } -} - impl Configurator for VercelBlobConfig { type Builder = VercelBlobBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/vercel_blob/config.rs b/core/src/services/vercel_blob/config.rs new file mode 100644 index 000000000000..241ee039e6a0 --- /dev/null +++ b/core/src/services/vercel_blob/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for VercelBlob services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct VercelBlobConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// vercel blob token. + pub token: String, +} + +impl Debug for VercelBlobConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Config"); + + ds.field("root", &self.root); + + ds.finish() + } +} diff --git a/core/src/services/vercel_blob/mod.rs b/core/src/services/vercel_blob/mod.rs index 2804d3fe7357..fa27ef304a78 100644 --- a/core/src/services/vercel_blob/mod.rs +++ b/core/src/services/vercel_blob/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::VercelBlobBuilder as VercelBlob; -pub use backend::VercelBlobConfig; - +#[cfg(feature = "services-vercel-blob")] mod core; +#[cfg(feature = "services-vercel-blob")] mod error; +#[cfg(feature = "services-vercel-blob")] mod lister; +#[cfg(feature = "services-vercel-blob")] mod writer; + +#[cfg(feature = "services-vercel-blob")] +mod backend; +#[cfg(feature = "services-vercel-blob")] +pub use backend::VercelBlobBuilder as VercelBlob; + +mod config; +pub use config::VercelBlobConfig; diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 94c303736241..e6f3b838b3f4 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -23,47 +23,15 @@ use std::sync::Arc; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; use super::lister::WebdavLister; use super::writer::WebdavWriter; use crate::raw::*; +use crate::services::WebdavConfig; use crate::*; -/// Config for [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct WebdavConfig { - /// endpoint of this backend - pub endpoint: Option, - /// username of this backend - pub username: Option, - /// password of this backend - pub password: Option, - /// token of this backend - pub token: Option, - /// root of this backend - pub root: Option, - /// WebDAV Service doesn't support copy. - pub disable_copy: bool, -} - -impl Debug for WebdavConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("WebdavConfig"); - - d.field("endpoint", &self.endpoint) - .field("username", &self.username) - .field("root", &self.root); - - d.finish_non_exhaustive() - } -} - impl Configurator for WebdavConfig { type Builder = WebdavBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/webdav/config.rs b/core/src/services/webdav/config.rs new file mode 100644 index 000000000000..95688c9a1666 --- /dev/null +++ b/core/src/services/webdav/config.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct WebdavConfig { + /// endpoint of this backend + pub endpoint: Option, + /// username of this backend + pub username: Option, + /// password of this backend + pub password: Option, + /// token of this backend + pub token: Option, + /// root of this backend + pub root: Option, + /// WebDAV Service doesn't support copy. + pub disable_copy: bool, +} + +impl Debug for WebdavConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("WebdavConfig"); + + d.field("endpoint", &self.endpoint) + .field("username", &self.username) + .field("root", &self.root); + + d.finish_non_exhaustive() + } +} diff --git a/core/src/services/webdav/mod.rs b/core/src/services/webdav/mod.rs index 1e18871a98c8..721f192613ee 100644 --- a/core/src/services/webdav/mod.rs +++ b/core/src/services/webdav/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::WebdavBuilder as Webdav; -pub use backend::WebdavConfig; - +#[cfg(feature = "services-webdav")] mod core; +#[cfg(feature = "services-webdav")] mod error; +#[cfg(feature = "services-webdav")] mod lister; +#[cfg(feature = "services-webdav")] mod writer; + +#[cfg(feature = "services-webdav")] +mod backend; +#[cfg(feature = "services-webdav")] +pub use backend::WebdavBuilder as Webdav; + +mod config; +pub use config::WebdavConfig; diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 0354f3659d4d..865eb14ed606 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -27,7 +27,6 @@ use http::Response; use http::StatusCode; use log::debug; use serde::Deserialize; -use serde::Serialize; use tokio::sync::OnceCell; use super::error::parse_error; @@ -38,37 +37,11 @@ use super::message::FileStatusWrapper; use super::writer::WebhdfsWriter; use super::writer::WebhdfsWriters; use crate::raw::*; +use crate::services::WebhdfsConfig; use crate::*; const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870"; -/// Config for WebHDFS support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct WebhdfsConfig { - /// Root for webhdfs. - pub root: Option, - /// Endpoint for webhdfs. - pub endpoint: Option, - /// Delegation token for webhdfs. - pub delegation: Option, - /// Disable batch listing - pub disable_list_batch: bool, - /// atomic_write_dir of this backend - pub atomic_write_dir: Option, -} - -impl Debug for WebhdfsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WebhdfsConfig") - .field("root", &self.root) - .field("endpoint", &self.endpoint) - .field("atomic_write_dir", &self.atomic_write_dir) - .finish_non_exhaustive() - } -} - impl Configurator for WebhdfsConfig { type Builder = WebhdfsBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/webhdfs/config.rs b/core/src/services/webhdfs/config.rs new file mode 100644 index 000000000000..6e7d8231303a --- /dev/null +++ b/core/src/services/webhdfs/config.rs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for WebHDFS support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct WebhdfsConfig { + /// Root for webhdfs. + pub root: Option, + /// Endpoint for webhdfs. + pub endpoint: Option, + /// Delegation token for webhdfs. + pub delegation: Option, + /// Disable batch listing + pub disable_list_batch: bool, + /// atomic_write_dir of this backend + pub atomic_write_dir: Option, +} + +impl Debug for WebhdfsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebhdfsConfig") + .field("root", &self.root) + .field("endpoint", &self.endpoint) + .field("atomic_write_dir", &self.atomic_write_dir) + .finish_non_exhaustive() + } +} diff --git a/core/src/services/webhdfs/mod.rs b/core/src/services/webhdfs/mod.rs index 30dda8b28f23..dc0e3faf851f 100644 --- a/core/src/services/webhdfs/mod.rs +++ b/core/src/services/webhdfs/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::WebhdfsBuilder as Webhdfs; -pub use backend::WebhdfsConfig; - +#[cfg(feature = "services-webhdfs")] mod error; +#[cfg(feature = "services-webhdfs")] mod lister; +#[cfg(feature = "services-webhdfs")] mod message; +#[cfg(feature = "services-webhdfs")] mod writer; + +#[cfg(feature = "services-webhdfs")] +mod backend; +#[cfg(feature = "services-webhdfs")] +pub use backend::WebhdfsBuilder as Webhdfs; + +mod config; +pub use config::WebhdfsConfig; diff --git a/core/src/services/yandex_disk/backend.rs b/core/src/services/yandex_disk/backend.rs index cbcbc7bca898..7b989fb9ad76 100644 --- a/core/src/services/yandex_disk/backend.rs +++ b/core/src/services/yandex_disk/backend.rs @@ -25,8 +25,6 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; -use serde::Deserialize; -use serde::Serialize; use super::core::*; use super::error::parse_error; @@ -34,31 +32,9 @@ use super::lister::YandexDiskLister; use super::writer::YandexDiskWriter; use super::writer::YandexDiskWriters; use crate::raw::*; +use crate::services::YandexDiskConfig; use crate::*; -/// Config for YandexDisk services support. -#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] -#[serde(default)] -#[non_exhaustive] -pub struct YandexDiskConfig { - /// root of this backend. - /// - /// All operations will happen under this root. - pub root: Option, - /// yandex disk oauth access_token. - pub access_token: String, -} - -impl Debug for YandexDiskConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Config"); - - ds.field("root", &self.root); - - ds.finish() - } -} - impl Configurator for YandexDiskConfig { type Builder = YandexDiskBuilder; fn into_builder(self) -> Self::Builder { diff --git a/core/src/services/yandex_disk/config.rs b/core/src/services/yandex_disk/config.rs new file mode 100644 index 000000000000..59cedb5e1a70 --- /dev/null +++ b/core/src/services/yandex_disk/config.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::{Debug, Formatter}; + +/// Config for YandexDisk services support. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct YandexDiskConfig { + /// root of this backend. + /// + /// All operations will happen under this root. + pub root: Option, + /// yandex disk oauth access_token. + pub access_token: String, +} + +impl Debug for YandexDiskConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Config"); + + ds.field("root", &self.root); + + ds.finish() + } +} diff --git a/core/src/services/yandex_disk/mod.rs b/core/src/services/yandex_disk/mod.rs index e2f2aff44888..bae1f563650c 100644 --- a/core/src/services/yandex_disk/mod.rs +++ b/core/src/services/yandex_disk/mod.rs @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. -mod backend; -pub use backend::YandexDiskBuilder as YandexDisk; -pub use backend::YandexDiskConfig; - +#[cfg(feature = "services-yandex-disk")] mod core; +#[cfg(feature = "services-yandex-disk")] mod error; +#[cfg(feature = "services-yandex-disk")] mod lister; +#[cfg(feature = "services-yandex-disk")] mod writer; + +#[cfg(feature = "services-yandex-disk")] +mod backend; +#[cfg(feature = "services-yandex-disk")] +pub use backend::YandexDiskBuilder as YandexDisk; + +mod config; +pub use config::YandexDiskConfig;