Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions src/aws/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ use crate::aws::{
};
use crate::client::{HttpConnector, TokenCredentialProvider, http_connector};
use crate::config::ConfigValue;
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use crate::{
Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider,
};
use base64::Engine;
use base64::prelude::BASE64_STANDARD;
use itertools::Itertools;
Expand Down Expand Up @@ -191,6 +193,8 @@ pub struct AmazonS3Builder {
request_payer: ConfigValue<RequesterPayer>,
/// The [`HttpConnector`] to use
http_connector: Option<Arc<dyn HttpConnector>>,
/// Capabilities to advertise for this store instance
capabilities: Option<Capabilities>,
}

/// Configuration keys for [`AmazonS3Builder`]
Expand Down Expand Up @@ -1071,6 +1075,17 @@ impl AmazonS3Builder {
self
}

/// Override the [`Capabilities`] advertised by this store.
///
/// By default the store reports `ordered_listing: true` because S3
/// `ListObjectsV2` returns results in lexicographic order. Use this
/// method if you are connecting to an S3-compatible endpoint whose
/// behaviour differs from the standard S3 API.
pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self {
self.capabilities = Some(capabilities);
self
}

/// Create a [`AmazonS3`] instance from the provided values,
/// consuming `self`.
pub fn build(mut self) -> Result<AmazonS3> {
Expand Down Expand Up @@ -1251,7 +1266,10 @@ impl AmazonS3Builder {
let http_client = http.connect(&config.client_options)?;
let client = Arc::new(S3Client::new(config, http_client));

Ok(AmazonS3 { client })
Ok(AmazonS3 {
client,
capabilities: self.capabilities,
})
}
}

Expand Down
13 changes: 10 additions & 3 deletions src/aws/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ use crate::multipart::{MultipartStore, PartId};
use crate::signer::Signer;
use crate::util::STRICT_ENCODE_SET;
use crate::{
CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload,
ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult,
Result, UploadPart,
Capabilities, Capability, CopyMode, CopyOptions, Error, GetOptions, GetResult, ListResult,
MultipartId, MultipartUpload, ObjectMeta, ObjectStore, Path, PutMode, PutMultipartOptions,
PutOptions, PutPayload, PutResult, Result, UploadPart,
};

static TAGS_HEADER: HeaderName = HeaderName::from_static("x-amz-tagging");
Expand Down Expand Up @@ -79,10 +79,13 @@ use crate::client::parts::Parts;
use crate::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore};
pub use credential::{AwsAuthorizer, AwsCredential};

const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([]);

/// Interface for [Amazon S3](https://aws.amazon.com/s3/).
#[derive(Debug, Clone)]
pub struct AmazonS3 {
client: Arc<S3Client>,
capabilities: Option<Capabilities>,
}

impl std::fmt::Display for AmazonS3 {
Expand Down Expand Up @@ -394,6 +397,10 @@ impl ObjectStore for AmazonS3 {
}
}
}

fn capabilities(&self) -> Capabilities {
self.capabilities.or(DEFAULT_CAPABILITIES)
}
}

#[derive(Debug)]
Expand Down
22 changes: 20 additions & 2 deletions src/azure/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ use crate::azure::credential::{
use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE};
use crate::client::{HttpConnector, TokenCredentialProvider, http_connector};
use crate::config::ConfigValue;
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use crate::{
Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider,
};
use percent_encoding::percent_decode_str;
use serde::{Deserialize, Serialize};
use std::str::FromStr;
Expand Down Expand Up @@ -180,6 +182,8 @@ pub struct MicrosoftAzureBuilder {
fabric_cluster_identifier: Option<String>,
/// The [`HttpConnector`] to use
http_connector: Option<Arc<dyn HttpConnector>>,
/// Capabilities to advertise for this store instance
capabilities: Option<Capabilities>,
}

/// Configuration keys for [`MicrosoftAzureBuilder`]
Expand Down Expand Up @@ -906,6 +910,17 @@ impl MicrosoftAzureBuilder {
self
}

/// Override the [`Capabilities`] advertised by this store.
///
/// By default the store reports `ordered_listing: true` because Azure Blob
/// Storage returns list results in lexicographic order. Use this method if
/// you are connecting to an endpoint whose behaviour differs from the
/// standard Azure Blob Storage API.
pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self {
self.capabilities = capabilities;
self
}

/// Configure a connection to container with given name on Microsoft Azure Blob store.
pub fn build(mut self) -> Result<MicrosoftAzure> {
if let Some(url) = self.url.take() {
Expand Down Expand Up @@ -1054,7 +1069,10 @@ impl MicrosoftAzureBuilder {
let http_client = http.connect(&config.client_options)?;
let client = Arc::new(AzureClient::new(config, http_client));

Ok(MicrosoftAzure { client })
Ok(MicrosoftAzure {
client,
capabilities: self.capabilities,
})
}
}

Expand Down
12 changes: 9 additions & 3 deletions src/azure/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
//! Unused blocks will automatically be dropped after 7 days.
//!
use crate::{
CopyMode, CopyOptions, GetOptions, GetResult, ListResult, MultipartId, MultipartUpload,
ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
UploadPart,
Capabilities, Capability, CopyMode, CopyOptions, GetOptions, GetResult, ListResult,
MultipartId, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions,
PutPayload, PutResult, Result, UploadPart,
multipart::{MultipartStore, PartId},
path::Path,
signer::Signer,
Expand Down Expand Up @@ -57,11 +57,13 @@ pub use builder::{AzureConfigKey, MicrosoftAzureBuilder};
pub use credential::AzureCredential;

const STORE: &str = "MicrosoftAzure";
const DEFAULT_CAPABILITIES: Capabilities = Capabilities::new([Capability::OrderedListing]);

/// Interface for [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/).
#[derive(Debug)]
pub struct MicrosoftAzure {
client: Arc<AzureClient>,
capabilities: Option<Capabilities>,
}

impl MicrosoftAzure {
Expand Down Expand Up @@ -180,6 +182,10 @@ impl ObjectStore for MicrosoftAzure {
CopyMode::Create => self.client.copy_request(from, to, false).await,
}
}

fn capabilities(&self) -> Capabilities {
self.capabilities.or(DEFAULT_CAPABILITIES)
}
}

#[async_trait]
Expand Down
143 changes: 143 additions & 0 deletions src/capabilities.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
use crate::Error;
use std::collections::HashSet;

/// An individual capability that an [`ObjectStore`] implementation may support.
///
/// Used together with [`Capabilities`] to advertise optional backend features.
/// Obtain the set of supported capabilities via [`ObjectStore::capabilities`].
///
/// # String representation
///
/// Each variant has a stable kebab-case string form accessible via
/// [`Capability::as_str`] and parseable via [`Capability::from_str`].
/// These strings are intended for configuration, logging, and serialisation.
#[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)]
pub enum Capability {
/// List results from [`ObjectStore::list`] and
/// [`ObjectStore::list_with_offset`] are returned in ascending
/// lexicographic order by [`Path`].
///
/// When this capability is present, callers may rely on the ordering and
/// avoid buffering all results solely for sorting purposes.
OrderedListing,
}

impl Capability {
/// Returns the stable kebab-case string representation of this capability.
///
/// The returned string can be round-tripped through [`Capability::from_str`].
pub fn as_str(&self) -> &'static str {
match self {
Capability::OrderedListing => "ordered-listing",
}
}

/// Parses a capability from its kebab-case string representation.
///
/// Returns `None` if `s` does not correspond to any known capability.
pub fn from_str(s: &str) -> Option<Self> {
match s {
"ordered-listing" => Some(Capability::OrderedListing),
_ => None,
}
}
}

/// Optional features supported by an [`ObjectStore`] implementation.
///
/// Obtain the capabilities of a store by calling [`ObjectStore::capabilities`].
/// All fields default to `false`; a store sets a field to `true` when it
/// natively supports that feature.
///
/// The struct is `#[non_exhaustive]` so that new capability flags can be added
/// in future versions without breaking existing code.
///
/// # Example
///
/// ```
/// # use object_store::{ObjectStore, memory::InMemory, Capability};
/// let store = InMemory::new();
/// if store.capabilities().has(Capability::OrderedListing) {
/// println!("list() results are in lexicographic order — no need to sort");
/// }
/// ```
#[derive(Debug, PartialEq)]
pub struct Capabilities {
supported: HashSet<Capability>,
}

impl Capabilities {
/// Create a [`Capabilities`] from an explicit list of supported [`Capability`] values.
///
/// Any capability not included in `capabilities` is considered unsupported.
///
/// # Example
///
/// ```
/// # use object_store::{Capabilities, Capability};
/// let caps = Capabilities::new([Capability::OrderedListing]);
/// assert!(caps.has(Capability::OrderedListing));
/// ```
pub fn new(capabilities: impl IntoIterator<Item = Capability>) -> Self {
Self {
supported: capabilities.into_iter().collect(),
}
}

/// Returns `true` if the given [`Capability`] is supported by this store.
pub fn has(&self, capability: Capability) -> bool {
self.supported.contains(&capability)
}

pub fn from_str(s: &str) -> crate::Result<Self> {
let mut capabilities: Vec<Capability> = Vec::new();
for mut cap in s.split(',') {
cap = cap.trim();
if cap.is_empty() {
continue;
}
match Capability::from_str(cap) {
Some(cap) => capabilities.push(cap),
None => {
return Err(Error::Generic {
store: "Config",
source: format!("invalid capability: {cap}").into(),
});
}
}
}
Ok(Self::new(capabilities))
}
}

#[cfg(test)]
mod tests {
use super::{Capabilities, Capability};

#[test]
fn test_capability() {
assert_eq!(Capability::OrderedListing.as_str(), "ordered-listing");
assert_eq!(
Capability::OrderedListing,
Capability::from_str("ordered-listing").unwrap()
);
assert_eq!(Capability::from_str("invalid").is_some(), false);
}

#[test]
fn test_capabilities() {
assert_eq!(Capabilities::from_str("invalid").is_err(), true);
assert_eq!(
Capabilities::from_str("")
.unwrap()
.has(Capability::OrderedListing),
false
);
assert_eq!(
Capabilities::from_str("ordered-listing")
.unwrap()
.has(Capability::OrderedListing),
true
);
}
}
8 changes: 7 additions & 1 deletion src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use std::time::Duration;
use humantime::{format_duration, parse_duration};
use reqwest::header::HeaderValue;

use crate::{Error, Result};
use crate::{Capabilities, Capability, Error, Result};

/// Provides deferred parsing of a value
///
Expand Down Expand Up @@ -121,6 +121,12 @@ impl Parse for HeaderValue {
}
}

impl Parse for Capabilities {
fn parse(v: &str) -> Result<Self> {
Self::from_str(v)
}
}

pub(crate) fn fmt_duration(duration: &ConfigValue<Duration>) -> String {
match duration {
ConfigValue::Parsed(v) => format_duration(*v).to_string(),
Expand Down
18 changes: 17 additions & 1 deletion src/gcp/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ use crate::gcp::{
GcpCredential, GcpCredentialProvider, GcpSigningCredential, GcpSigningCredentialProvider,
GoogleCloudStorage, STORE, credential,
};
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use crate::{
Capabilities, ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider,
};
use serde::{Deserialize, Serialize};
use std::str::FromStr;
use std::sync::Arc;
Expand Down Expand Up @@ -118,6 +120,8 @@ pub struct GoogleCloudStorageBuilder {
signing_credentials: Option<GcpSigningCredentialProvider>,
/// The [`HttpConnector`] to use
http_connector: Option<Arc<dyn HttpConnector>>,
/// Capabilities to advertise for this store instance
capabilities: Option<Capabilities>,
}

/// Configuration keys for [`GoogleCloudStorageBuilder`]
Expand Down Expand Up @@ -499,6 +503,17 @@ impl GoogleCloudStorageBuilder {
self
}

/// Override the [`Capabilities`] advertised by this store.
///
/// By default the store reports `ordered_listing: true` because GCS
/// returns list results in lexicographic order. Use this method if you
/// are connecting to an endpoint whose behaviour differs from the
/// standard GCS API.
pub fn with_capabilities(mut self, capabilities: Capabilities) -> Self {
self.capabilities = Some(capabilities);
self
}

/// Configure a connection to Google Cloud Storage, returning a
/// new [`GoogleCloudStorage`] and consuming `self`
pub fn build(mut self) -> Result<GoogleCloudStorage> {
Expand Down Expand Up @@ -634,6 +649,7 @@ impl GoogleCloudStorageBuilder {
let http_client = http.connect(&config.client_options)?;
Ok(GoogleCloudStorage {
client: Arc::new(GoogleCloudStorageClient::new(config, http_client)?),
capabilities: self.capabilities,
})
}
}
Expand Down
Loading