diff --git a/CHANGELOG.md b/CHANGELOG.md index 538af1e6..7e69ad13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `Node::async_new_with_store` and implement `storage::async_get_child_nodes` - Add `async_array_write_read` and `async_http_array_read` examples - Add experimental `async` feature (disabled by default) + - Add experimental async `Amazon S3`, `Google Cloud`, `Microsoft Azure` stores (untested) ### Changed - Bump itertools to `0.12` and zstd to `0.13` diff --git a/Cargo.toml b/Cargo.toml index 0b2ed947..7d7b02da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,10 @@ transpose = ["dep:ndarray"] zfp = ["dep:zfp-sys"] zstd = ["dep:zstd"] # Stores -http = ["dep:reqwest", "dep:url"] +http = ["dep:reqwest", "dep:url", "object_store/http"] +s3 = ["object_store/aws"] +gcp = ["object_store/gcp"] +azure = ["object_store/azure"] zip = ["dep:zip"] # Misc ndarray = ["dep:ndarray"] # Adds ndarray utility functions to Array @@ -53,7 +56,7 @@ inventory = "0.3" itertools = "0.12" ndarray = { version = "0.15", optional = true } num = { version = "0.4" } -object_store = { version = "0.8.0", features = ["http"] } +object_store = { version = "0.8.0", optional = true } parking_lot = "0.12" pathdiff = "0.2" rayon = "1.6" diff --git a/TODO.md b/TODO.md index 2a49ffb0..382183f3 100644 --- a/TODO.md +++ b/TODO.md @@ -2,5 +2,4 @@ - Review documentation - Increase test coverage - URI support [see ZEP0008](https://github.com/zarr-developers/zeps/pull/48) -- Investigate the [object_store](https://docs.rs/object_store/latest/object_store/) crate for store abstraction - Add a performance comparison to other Zarr implementations diff --git a/src/lib.rs b/src/lib.rs index a49b1ebd..4123cff8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,7 +17,7 @@ //! - [x] [ZEP0003 - Variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html) ([draft](https://github.com/orgs/zarr-developers/discussions/52)) //! - [x] Stores //! - Sync: [`filesystem`](crate::storage::store::FilesystemStore), [`memory`](crate::storage::store::MemoryStore), [`http`](crate::storage::store::HTTPStore), [`zip`](crate::storage::storage_adapter::ZipStorageAdapter) -//! - Async: [`filesystem`](crate::storage::store::AsyncFilesystemStore), [`memory`](crate::storage::store::AsyncMemoryStore), [`http`](crate::storage::store::AsyncHTTPStore) +//! - Async: [`filesystem`](crate::storage::store::AsyncFilesystemStore), [`memory`](crate::storage::store::AsyncMemoryStore), [HTTP](crate::storage::store::AsyncHTTPStore), [Google Cloud Storage](crate::storage::store::AsyncGoogleCloudStore), [Amazon s3](crate::storage::store::AsyncAmazonS3Store), [Microsoft Azure Storage](crate::storage::store::AsyncMicrosoftAzureStore) //! - [x] Data types: [core data types](crate::array::data_type::DataType), [`raw bits`](crate::array::data_type::DataType::RawBits), [`float16`](crate::array::data_type::DataType::Float16), [`bfloat16`](crate::array::data_type::DataType::BFloat16) [(spec issue)](https://github.com/zarr-developers/zarr-specs/issues/130) //! - [x] Chunk grids: [`regular`](crate::array::chunk_grid::RegularChunkGrid), [`rectangular`](crate::array::chunk_grid::RectangularChunkGrid) ([draft](https://github.com/orgs/zarr-developers/discussions/52)) //! - [x] Chunk key encoding: [`default`](crate::array::chunk_key_encoding::DefaultChunkKeyEncoding), [`v2`](crate::array::chunk_key_encoding::V2ChunkKeyEncoding) @@ -35,6 +35,7 @@ //! //! The following features are disabled by default: //! - Codecs: `bitround`, `zfp` +//! - Stores: `s3` (Amazon S3) `gcp` (Google Cloud), `azure` (Microsoft Azure) //! - `async` (experimental): enable asynchronous stores and associated storage, array, and group methods. //! //! ## Examples diff --git a/src/storage/store.rs b/src/storage/store.rs index e93a176d..418ccc9e 100644 --- a/src/storage/store.rs +++ b/src/storage/store.rs @@ -23,6 +23,15 @@ pub use r#async::http_store::AsyncHTTPStore; #[cfg(feature = "http")] pub use sync::http_store::{HTTPStore, HTTPStoreCreateError}; +#[cfg(all(feature = "async", feature = "s3"))] +pub use r#async::amazon_s3_store::AsyncAmazonS3Store; + +#[cfg(all(feature = "async", feature = "gcp"))] +pub use r#async::google_cloud_store::AsyncGoogleCloudStore; + +#[cfg(all(feature = "async", feature = "azure"))] +pub use r#async::microsoft_azure_store::AsyncMicrosoftAzureStore; + // pub use store_plugin::{StorePlugin, StorePluginCreateError}; // Currently disabled. use std::sync::Arc; diff --git a/src/storage/store/async.rs b/src/storage/store/async.rs index 0b7ccd67..a4c239be 100644 --- a/src/storage/store/async.rs +++ b/src/storage/store/async.rs @@ -2,5 +2,11 @@ pub mod filesystem_store; pub mod memory_store; pub mod object_store; +#[cfg(feature = "s3")] +pub mod amazon_s3_store; +#[cfg(feature = "gcp")] +pub mod google_cloud_store; #[cfg(feature = "http")] pub mod http_store; +#[cfg(feature = "azure")] +pub mod microsoft_azure_store; diff --git a/src/storage/store/async/amazon_s3_store.rs b/src/storage/store/async/amazon_s3_store.rs new file mode 100644 index 00000000..c3db9685 --- /dev/null +++ b/src/storage/store/async/amazon_s3_store.rs @@ -0,0 +1,18 @@ +//! Amazon S3 stores. + +use crate::{object_store_impl, storage::StorageError}; + +/// An Amazon S3 store. +#[derive(Debug)] +pub struct AsyncAmazonS3Store { + object_store: object_store::aws::AmazonS3, +} + +impl AsyncAmazonS3Store { + /// Create a new amazon S3 store. + pub fn new(object_store: object_store::aws::AmazonS3) -> Self { + Self { object_store } + } +} + +object_store_impl!(AsyncAmazonS3Store, object_store); diff --git a/src/storage/store/async/google_cloud_store.rs b/src/storage/store/async/google_cloud_store.rs new file mode 100644 index 00000000..f5828950 --- /dev/null +++ b/src/storage/store/async/google_cloud_store.rs @@ -0,0 +1,18 @@ +//! Google Cloud stores. + +use crate::{object_store_impl, storage::StorageError}; + +/// A Google Cloud Storage store. +#[derive(Debug)] +pub struct AsyncGoogleCloudStore { + object_store: object_store::gcp::GoogleCloudStorage, +} + +impl AsyncGoogleCloudStore { + /// Create a new amazon S3 store. + pub fn new(object_store: object_store::gcp::GoogleCloudStorage) -> Self { + Self { object_store } + } +} + +object_store_impl!(AsyncGoogleCloudStore, object_store); diff --git a/src/storage/store/async/microsoft_azure_store.rs b/src/storage/store/async/microsoft_azure_store.rs new file mode 100644 index 00000000..69d47739 --- /dev/null +++ b/src/storage/store/async/microsoft_azure_store.rs @@ -0,0 +1,18 @@ +//! Azure blob storage stores. + +use crate::{object_store_impl, storage::StorageError}; + +/// A Microsoft Azure store. +#[derive(Debug)] +pub struct AsyncMicrosoftAzureStore { + object_store: object_store::azure::MicrosoftAzure, +} + +impl AsyncMicrosoftAzureStore { + /// Create a new amazon S3 store. + pub fn new(object_store: object_store::azure::MicrosoftAzure) -> Self { + Self { object_store } + } +} + +object_store_impl!(AsyncMicrosoftAzureStore, object_store);