diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml index cc870c87e..eaf7365d9 100644 --- a/ballista/client/Cargo.toml +++ b/ballista/client/Cargo.toml @@ -45,4 +45,5 @@ default = [] hdfs = ["ballista-core/hdfs"] hdfs3 = ["ballista-core/hdfs3"] s3 = ["ballista-core/s3"] +azure = ["ballista-core/azure"] standalone = ["ballista-executor", "ballista-scheduler"] diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml index 0d8621f07..d39675e03 100644 --- a/ballista/core/Cargo.toml +++ b/ballista/core/Cargo.toml @@ -40,6 +40,7 @@ force_hash_collisions = ["datafusion/force_hash_collisions"] hdfs = ["datafusion-objectstore-hdfs/hdfs"] hdfs3 = ["datafusion-objectstore-hdfs/hdfs3"] s3 = ["object_store/aws"] +azure = ["object_store/azure"] simd = ["datafusion/simd"] [dependencies] diff --git a/ballista/core/src/utils.rs b/ballista/core/src/utils.rs index 9fb62cd2d..c83e94aa8 100644 --- a/ballista/core/src/utils.rs +++ b/ballista/core/src/utils.rs @@ -52,6 +52,8 @@ use futures::StreamExt; use log::error; #[cfg(feature = "s3")] use object_store::aws::AmazonS3Builder; +#[cfg(feature = "azure")] +use object_store::azure::MicrosoftAzureBuilder; use object_store::ObjectStore; use std::io::{BufWriter, Write}; use std::marker::PhantomData; @@ -120,6 +122,18 @@ impl ObjectStoreProvider for FeatureBasedObjectStoreProvider { } } + #[cfg(feature = "azure")] + { + if url.to_string().starts_with("azure://") { + if let Some(bucket_name) = url.host_str() { + let store = MicrosoftAzureBuilder::from_env() + .with_container_name(bucket_name) + .build()?; + return Ok(Arc::new(store)); + } + } + } + Err(DataFusionError::Execution(format!( "No object store available for {}", url