feat: Add write_total_max_size in Capability

core/Cargo.toml

@@ -115,6 +115,7 @@ services-azdls = [
   "reqsign?/services-azblob",
   "reqsign?/reqwest_request",
 ]
+services-azfile = []
 services-cacache = ["dep:cacache"]
 services-cloudflare-kv = []
 services-cos = [
@@ -145,6 +146,7 @@ services-memcached = ["dep:bb8"]
 services-memory = []
 services-mini-moka = ["dep:mini-moka"]
 services-moka = ["dep:moka"]
+services-mongodb = ["dep:mongodb"]
 services-mysql = ["dep:mysql_async"]
 services-obs = [
   "dep:reqsign",
@@ -182,8 +184,6 @@ services-wasabi = [
 ]
 services-webdav = []
 services-webhdfs = []
-services-azfile = []
-services-mongodb = ["dep:mongodb"]
 
 [lib]
 bench = false
@@ -243,6 +243,7 @@ metrics = { version = "0.20", optional = true }
 mini-moka = { version = "0.10", optional = true }
 minitrace = { version = "0.5", optional = true }
 moka = { version = "0.10", optional = true, features = ["future"] }
+mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
 mysql_async = { version = "0.32.2", optional = true }
 once_cell = "1"
 openssh = { version = "0.9.9", optional = true }
@@ -286,7 +287,6 @@ tokio = "1.27"
 tokio-postgres = { version = "0.7.8", optional = true }
 tracing = { version = "0.1", optional = true }
 uuid = { version = "1", features = ["serde", "v4"] }
-mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
 
 [dev-dependencies]
 criterion = { version = "0.4", features = ["async", "async_tokio"] }

core/src/services/d1/backend.rs

@@ -271,6 +271,9 @@ impl kv::Adapter for Adapter {
             Capability {
                 read: true,
                 write: true,
+                // Cloudflare D1 supports 1MB as max in write_total.
+                // refer to https://developers.cloudflare.com/d1/platform/limits/
+                write_total_max_size: Some(1000 * 1000),
                 ..Default::default()
             },
         )

core/src/types/capability.rs

@@ -98,6 +98,10 @@ pub struct Capability {
     ///
     /// For example, Google GCS requires align size to 256KiB in write_multi.
     pub write_multi_align_size: Option<usize>,
+    /// write_total_max_size is the max size that services support in write_total.
+    ///
+    /// For example, Cloudflare D1 supports 1MB as max in write_total.
+    pub write_total_max_size: Option<usize>,
 
     /// If operator supports create dir.
     pub create_dir: bool,

core/src/types/operator/builder.rs

@@ -163,6 +163,8 @@ impl Operator {
             Scheme::Cacache => Self::from_map::<services::Cacache>(map)?.finish(),
             #[cfg(feature = "services-cos")]
             Scheme::Cos => Self::from_map::<services::Cos>(map)?.finish(),
+            #[cfg(feature = "services-d1")]
+            Scheme::D1 => Self::from_map::<services::D1>(map)?.finish(),
             #[cfg(feature = "services-dashmap")]
             Scheme::Dashmap => Self::from_map::<services::Dashmap>(map)?.finish(),
             #[cfg(feature = "services-dropbox")]

core/tests/behavior/append.rs

@@ -47,8 +47,8 @@ pub fn behavior_append_tests(op: &Operator) -> Vec<Trial> {
 /// Test append to a file must success.
 pub async fn test_append_create_append(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content_one, size_one) = gen_bytes();
-    let (content_two, size_two) = gen_bytes();
+    let (content_one, size_one) = gen_bytes(op.info().full_capability());
+    let (content_two, size_two) = gen_bytes(op.info().full_capability());
 
     op.write_with(&path, content_one.clone())
         .append(true)
@@ -74,7 +74,7 @@ pub async fn test_append_create_append(op: Operator) -> Result<()> {
 /// Test append to a directory path must fail.
 pub async fn test_append_with_dir_path(op: Operator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let res = op.write_with(&path, content).append(true).await;
     assert!(res.is_err());
@@ -90,7 +90,7 @@ pub async fn test_append_with_cache_control(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let target_cache_control = "no-cache, no-store, max-age=300";
     op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub async fn test_append_with_content_type(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_type = "application/json";
     op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub async fn test_append_with_content_disposition(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_disposition = "attachment; filename=\"filename.jpg\"";
     op.write_with(&path, content)

core/tests/behavior/blocking_append.rs

@@ -47,8 +47,8 @@ pub fn behavior_blocking_append_tests(op: &Operator) -> Vec<Trial> {
 /// Test append to a file must success.
 pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content_one, size_one) = gen_bytes();
-    let (content_two, size_two) = gen_bytes();
+    let (content_one, size_one) = gen_bytes(op.info().full_capability());
+    let (content_two, size_two) = gen_bytes(op.info().full_capability());
 
     op.write_with(&path, content_one.clone())
         .append(true)
@@ -74,7 +74,7 @@ pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
 /// Test append to a directory path must fail.
 pub fn test_blocking_append_with_dir_path(op: BlockingOperator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let res = op.write_with(&path, content).append(true).call();
     assert!(res.is_err());
@@ -90,7 +90,7 @@ pub fn test_blocking_append_with_cache_control(op: BlockingOperator) -> Result<(
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let target_cache_control = "no-cache, no-store, max-age=300";
     op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub fn test_blocking_append_with_content_type(op: BlockingOperator) -> Result<()
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_type = "application/json";
     op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub fn test_blocking_append_with_content_disposition(op: BlockingOperator) -> Re
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_disposition = "attachment; filename=\"filename.jpg\"";
     op.write_with(&path, content)

core/tests/behavior/blocking_copy.rs

@@ -41,7 +41,7 @@ pub fn behavior_blocking_copy_tests(op: &Operator) -> Vec<Trial> {
 /// Copy a file and test with stat.
 pub fn test_blocking_copy_file(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -86,7 +86,7 @@ pub fn test_blocking_copy_source_dir(op: BlockingOperator) -> Result<()> {
 /// Copy to a dir should return an error.
 pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -107,7 +107,7 @@ pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
 /// Copy a file to self should return an error.
 pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _size) = gen_bytes();
+    let (source_content, _size) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -123,7 +123,7 @@ pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
 /// Copy to a nested path, parent path should be created successfully.
 pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -147,12 +147,12 @@ pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
 /// Copy to a exist path should overwrite successfully.
 pub fn test_blocking_copy_overwrite(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content)?;

core/tests/behavior/blocking_list.rs

@@ -46,7 +46,7 @@ pub fn test_blocking_list_dir(op: BlockingOperator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -74,7 +74,7 @@ pub fn test_blocking_list_dir_with_metakey(op: BlockingOperator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -124,7 +124,7 @@ pub fn test_blocking_list_dir_with_metakey_complete(op: BlockingOperator) -> Res
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 

core/tests/behavior/blocking_rename.rs

@@ -41,7 +41,7 @@ pub fn behavior_blocking_rename_tests(op: &Operator) -> Vec<Trial> {
 /// Rename a file and test with stat.
 pub fn test_blocking_rename_file(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -89,7 +89,7 @@ pub fn test_blocking_rename_source_dir(op: BlockingOperator) -> Result<()> {
 /// Rename to a dir should return an error.
 pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -110,7 +110,7 @@ pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
 /// Rename a file to self should return an error.
 pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _size) = gen_bytes();
+    let (source_content, _size) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -126,7 +126,7 @@ pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
 /// Rename to a nested path, parent path should be created successfully.
 pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -153,12 +153,12 @@ pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
 /// Rename to a exist path should overwrite successfully.
 pub fn test_blocking_rename_overwrite(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content)?;