diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index fe49471c4907..17d779ff6a51 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -604,6 +604,9 @@ pub enum AmazonS3ConfigKey { /// - `aws_profile` /// - `profile` Profile, + + /// Client options + Client(ClientConfigKey), } impl AsRef for AmazonS3ConfigKey { @@ -622,6 +625,7 @@ impl AsRef for AmazonS3ConfigKey { Self::Profile => "aws_profile", Self::UnsignedPayload => "aws_unsigned_payload", Self::Checksum => "aws_checksum_algorithm", + Self::Client(opt) => opt.as_ref(), } } } @@ -652,7 +656,12 @@ impl FromStr for AmazonS3ConfigKey { "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint), "aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload), "aws_checksum_algorithm" | "checksum_algorithm" => Ok(Self::Checksum), - _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + // Backwards compatibility + "aws_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)), + _ => match s.parse() { + Ok(key) => Ok(Self::Client(key)), + Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + }, } } } @@ -688,9 +697,7 @@ impl AmazonS3Builder { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { if key.starts_with("AWS_") { - if let Ok(config_key) = - AmazonS3ConfigKey::from_str(&key.to_ascii_lowercase()) - { + if let Ok(config_key) = key.to_ascii_lowercase().parse() { builder = builder.with_config(config_key, value); } } @@ -706,12 +713,6 @@ impl AmazonS3Builder { Some(format!("{METADATA_ENDPOINT}{metadata_relative_uri}")); } - if let Ok(text) = std::env::var("AWS_ALLOW_HTTP") { - builder.client_options = builder - .client_options - .with_config(ClientConfigKey::AllowHttp, text); - } - builder } @@ -770,6 +771,9 @@ impl AmazonS3Builder { AmazonS3ConfigKey::Checksum => { self.checksum_algorithm = Some(ConfigValue::Deferred(value.into())) } + AmazonS3ConfigKey::Client(key) => { + self.client_options = self.client_options.with_config(key, value) + } }; self } @@ -834,6 +838,7 @@ impl AmazonS3Builder { AmazonS3ConfigKey::Checksum => { self.checksum_algorithm.as_ref().map(ToString::to_string) } + AmazonS3ConfigKey::Client(key) => self.client_options.get_config_value(key), } } diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs index 2b5b43adabe0..c2cfdfe6af32 100644 --- a/object_store/src/azure/mod.rs +++ b/object_store/src/azure/mod.rs @@ -559,6 +559,9 @@ pub enum AzureConfigKey { /// - `azure_use_azure_cli` /// - `use_azure_cli` UseAzureCli, + + /// Client options + Client(ClientConfigKey), } impl AsRef for AzureConfigKey { @@ -577,6 +580,7 @@ impl AsRef for AzureConfigKey { Self::MsiResourceId => "azure_msi_resource_id", Self::FederatedTokenFile => "azure_federated_token_file", Self::UseAzureCli => "azure_use_azure_cli", + Self::Client(key) => key.as_ref(), } } } @@ -621,7 +625,12 @@ impl FromStr for AzureConfigKey { Ok(Self::FederatedTokenFile) } "azure_use_azure_cli" | "use_azure_cli" => Ok(Self::UseAzureCli), - _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + // Backwards compatibility + "azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)), + _ => match s.parse() { + Ok(key) => Ok(Self::Client(key)), + Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + }, } } } @@ -664,21 +673,13 @@ impl MicrosoftAzureBuilder { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { if key.starts_with("AZURE_") { - if let Ok(config_key) = - AzureConfigKey::from_str(&key.to_ascii_lowercase()) - { + if let Ok(config_key) = key.to_ascii_lowercase().parse() { builder = builder.with_config(config_key, value); } } } } - if let Ok(text) = std::env::var("AZURE_ALLOW_HTTP") { - builder.client_options = builder - .client_options - .with_config(ClientConfigKey::AllowHttp, text) - } - if let Ok(text) = std::env::var(MSI_ENDPOINT_ENV_KEY) { builder = builder.with_msi_endpoint(text); } @@ -731,6 +732,9 @@ impl MicrosoftAzureBuilder { } AzureConfigKey::UseAzureCli => self.use_azure_cli.parse(value), AzureConfigKey::UseEmulator => self.use_emulator.parse(value), + AzureConfigKey::Client(key) => { + self.client_options = self.client_options.with_config(key, value) + } }; self } @@ -786,6 +790,7 @@ impl MicrosoftAzureBuilder { AzureConfigKey::MsiResourceId => self.msi_resource_id.clone(), AzureConfigKey::FederatedTokenFile => self.federated_token_file.clone(), AzureConfigKey::UseAzureCli => Some(self.use_azure_cli.to_string()), + AzureConfigKey::Client(key) => self.client_options.get_config_value(key), } } diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs index d7b0b86d99e5..d2242dd41089 100644 --- a/object_store/src/client/mod.rs +++ b/object_store/src/client/mod.rs @@ -31,6 +31,7 @@ use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::{Client, ClientBuilder, Proxy}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::str::FromStr; use std::time::Duration; use crate::path::Path; @@ -53,6 +54,28 @@ pub enum ClientConfigKey { AllowHttp, } +impl AsRef for ClientConfigKey { + fn as_ref(&self) -> &str { + match self { + Self::AllowHttp => "allow_http", + } + } +} + +impl FromStr for ClientConfigKey { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + match s { + "allow_http" => Ok(Self::AllowHttp), + _ => Err(super::Error::UnknownConfigurationKey { + store: "HTTP", + key: s.into(), + }), + } + } +} + /// HTTP client configuration for remote object stores #[derive(Debug, Clone, Default)] pub struct ClientOptions { diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs index 6f3d53d42f34..375b4d8f8c37 100644 --- a/object_store/src/gcp/mod.rs +++ b/object_store/src/gcp/mod.rs @@ -49,6 +49,7 @@ use url::Url; use crate::client::pagination::stream_paginated; use crate::client::retry::RetryExt; +use crate::client::ClientConfigKey; use crate::{ client::token::TokenCache, multipart::{CloudMultiPartUpload, CloudMultiPartUploadImpl, UploadPart}, @@ -829,6 +830,9 @@ pub enum GoogleConfigKey { /// /// See [`GoogleCloudStorageBuilder::with_application_credentials`]. ApplicationCredentials, + + /// Client options + Client(ClientConfigKey), } impl AsRef for GoogleConfigKey { @@ -838,6 +842,7 @@ impl AsRef for GoogleConfigKey { Self::ServiceAccountKey => "google_service_account_key", Self::Bucket => "google_bucket", Self::ApplicationCredentials => "google_application_credentials", + Self::Client(key) => key.as_ref(), } } } @@ -858,7 +863,10 @@ impl FromStr for GoogleConfigKey { Ok(Self::Bucket) } "google_application_credentials" => Ok(Self::ApplicationCredentials), - _ => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + _ => match s.parse() { + Ok(key) => Ok(Self::Client(key)), + Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()), + }, } } } @@ -911,9 +919,7 @@ impl GoogleCloudStorageBuilder { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) { if key.starts_with("GOOGLE_") { - if let Ok(config_key) = - GoogleConfigKey::from_str(&key.to_ascii_lowercase()) - { + if let Ok(config_key) = key.to_ascii_lowercase().parse() { builder = builder.with_config(config_key, value); } } @@ -957,6 +963,9 @@ impl GoogleCloudStorageBuilder { GoogleConfigKey::ApplicationCredentials => { self.application_credentials_path = Some(value.into()) } + GoogleConfigKey::Client(key) => { + self.client_options = self.client_options.with_config(key, value) + } }; self } @@ -1005,6 +1014,7 @@ impl GoogleCloudStorageBuilder { GoogleConfigKey::ApplicationCredentials => { self.application_credentials_path.clone() } + GoogleConfigKey::Client(key) => self.client_options.get_config_value(key), } }