Skip to content

fix(storage): shorten background s3 timeouts #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fewensa merged 1 commit into from
Aug 16, 2026
Merged

fix(storage): shorten background s3 timeouts #461

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions crates/storage/src/object_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,7 @@ const DEFAULT_S3_MAX_CONCURRENT_OPERATIONS: usize = 16;
const DEFAULT_S3_BACKGROUND_RUNTIME_WORKER_THREADS: usize = 4;
const DEFAULT_S3_BACKGROUND_MAX_CONCURRENT_OPERATIONS: usize = 16;
const DEFAULT_S3_OPERATION_TIMEOUT_MS: u64 = 30_000;
const DEFAULT_S3_BACKGROUND_OPERATION_TIMEOUT_MS: u64 = 5_000;
const DEFAULT_S3_GET_MAX_ATTEMPTS: usize = 3;
const DEFAULT_S3_GET_RETRY_BACKOFF_MS: u64 = 100;
const S3_SLOW_OPERATION_LOG_THRESHOLD_MS: u128 = 1_000;
Expand All @@ -830,6 +831,13 @@ impl S3OperationClass {
Self::Background => "background",
}
}

fn operation_timeout(self) -> Duration {
match self {
Self::Foreground => Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
Self::Background => Duration::from_millis(DEFAULT_S3_BACKGROUND_OPERATION_TIMEOUT_MS),
}
}
}

fn s3_operation_timeout_error_kind(operation: &str) -> DatalensErrorKind {
Expand Down Expand Up @@ -1056,7 +1064,7 @@ impl S3ObjectStore {
operation_class,
"config_load",
config.bucket.clone(),
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
operation_class.operation_timeout(),
async move {
let mut loader =
aws_config::defaults(BehaviorVersion::latest()).region(Region::new(region));
Expand Down Expand Up @@ -1112,7 +1120,7 @@ impl ObjectStore for S3ObjectStore {
let log_key = key.clone();
let client = self.client.clone();
let bucket = self.bucket.clone();
self.runtime().block_on_operation(self.operation_class, "get", log_key, Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS), async move {
self.runtime().block_on_operation(self.operation_class, "get", log_key, self.operation_class.operation_timeout(), async move {
for attempt in 1..=DEFAULT_S3_GET_MAX_ATTEMPTS {
let object = match client.get_object().bucket(&bucket).key(&key).send().await {
Ok(object) => object,
Expand Down Expand Up @@ -1174,7 +1182,7 @@ impl ObjectStore for S3ObjectStore {
let client = self.client.clone();
let bucket = self.bucket.clone();
self.runtime()
.block_on_operation(self.operation_class, "get_optional", log_key, Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS), async move {
.block_on_operation(self.operation_class, "get_optional", log_key, self.operation_class.operation_timeout(), async move {
for attempt in 1..=DEFAULT_S3_GET_MAX_ATTEMPTS {
let object = match client.get_object().bucket(&bucket).key(&key).send().await {
Ok(object) => object,
Expand Down Expand Up @@ -1237,7 +1245,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"put",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
client
.put_object()
Expand Down Expand Up @@ -1271,7 +1279,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"put_if_absent",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
match client
.put_object()
Expand Down Expand Up @@ -1304,7 +1312,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"exists",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
match client.head_object().bucket(&bucket).key(&key).send().await {
Ok(_) => Ok(true),
Expand All @@ -1327,7 +1335,7 @@ impl ObjectStore for S3ObjectStore {
let client = self.client.clone();
let bucket = self.bucket.clone();
self.runtime()
.block_on_operation(self.operation_class, "list", log_key, Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS), async move {
.block_on_operation(self.operation_class, "list", log_key, self.operation_class.operation_timeout(), async move {
let mut continuation = None;
let mut objects = Vec::new();
loop {
Expand Down Expand Up @@ -1414,7 +1422,7 @@ impl ObjectStore for S3ObjectStore {
let client = self.client.clone();
let bucket = self.bucket.clone();
self.runtime()
.block_on_operation(self.operation_class, "list_page", log_key, Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS), async move {
.block_on_operation(self.operation_class, "list_page", log_key, self.operation_class.operation_timeout(), async move {
let mut response = None;
for attempt in 1..=DEFAULT_S3_GET_MAX_ATTEMPTS {
match client
Expand Down Expand Up @@ -1484,7 +1492,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"delete",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
client
.delete_object()
Expand Down Expand Up @@ -1518,7 +1526,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"try_acquire_lock",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
match client
.put_object()
Expand Down Expand Up @@ -1556,7 +1564,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"release_lock",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
let object = match client
.get_object()
Expand Down Expand Up @@ -1626,7 +1634,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"renew_lock",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
let object = match client
.get_object()
Expand Down Expand Up @@ -1707,7 +1715,7 @@ impl ObjectStore for S3ObjectStore {
self.operation_class,
"takeover_expired_lock",
log_key,
Duration::from_millis(DEFAULT_S3_OPERATION_TIMEOUT_MS),
self.operation_class.operation_timeout(),
async move {
let object = match client
.get_object()
Expand Down Expand Up @@ -1947,6 +1955,18 @@ mod tests {
assert!(start.elapsed() < Duration::from_secs(1));
}

#[test]
fn test_s3_background_operations_have_shorter_timeout() {
assert_eq!(
S3OperationClass::Foreground.operation_timeout(),
Duration::from_secs(30)
);
assert_eq!(
S3OperationClass::Background.operation_timeout(),
Duration::from_secs(5)
);
}

#[test]
fn test_s3_get_retry_stops_after_max_attempts() {
assert!(should_retry_s3_get(1));
Expand Down
Loading