Skip to content

Fix all worker cleanup races by naming the workdir differently for each retry of an operation [1.6-patch-4b] #2646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cormacrelf wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix all worker cleanup races by naming the workdir differently for each retry of an operation [1.6-patch-4b] #2646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca847bb
Add test for action-drop-cleanup deleting live files
cormacrelf Jul 13, 2026
1ba1cb8
Add tests for issue 2001
cormacrelf Jul 31, 2026
4a9b88d
Use different working directory for each attempt at an operation
cormacrelf Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions nativelink-config/src/cas_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -909,17 +909,13 @@ pub struct LocalWorkerConfig {
)]
pub max_upload_timeout_s: usize,

/// Maximum time to wait for action directory cleanup before timing out.
/// Value in seconds.
///
/// Default: 30 seconds
/// Deprecated and ignored. Each action attempt now runs in its own
/// directory, so a retry never waits for a previous attempt's cleanup.
/// Retained so that existing configs continue to load.
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
pub max_cleanup_wait_s: usize,

/// Maximum backoff duration for exponential backoff when waiting for cleanup.
/// Value in milliseconds.
///
/// Default: 500 milliseconds
/// Deprecated and ignored. See `max_cleanup_wait_s`.
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
pub max_cleanup_backoff_ms: usize,

Expand Down
21 changes: 7 additions & 14 deletions nativelink-worker/src/local_worker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,6 @@ const DEFAULT_ENDPOINT_TIMEOUT_S: f32 = 5.;
/// If this value gets modified the documentation in `cas_server.rs` must also be updated.
const DEFAULT_MAX_ACTION_TIMEOUT: Duration = Duration::from_mins(20);
const DEFAULT_MAX_UPLOAD_TIMEOUT: Duration = Duration::from_mins(10);
const DEFAULT_MAX_CLEANUP_WAIT: Duration = Duration::from_secs(30);
const DEFAULT_MAX_CLEANUP_BACKOFF: Duration = Duration::from_millis(500);

struct FinishedActionResult {
action_result: ActionResult,
Expand Down Expand Up @@ -595,16 +593,13 @@ pub async fn new_local_worker(
} else {
Duration::from_secs(config.max_upload_timeout_s as u64)
};
let max_cleanup_wait = if config.max_cleanup_wait_s == 0 {
DEFAULT_MAX_CLEANUP_WAIT
} else {
Duration::from_secs(config.max_cleanup_wait_s as u64)
};
let max_cleanup_backoff = if config.max_cleanup_backoff_ms == 0 {
DEFAULT_MAX_CLEANUP_BACKOFF
} else {
Duration::from_millis(config.max_cleanup_backoff_ms as u64)
};
if config.max_cleanup_wait_s != 0 || config.max_cleanup_backoff_ms != 0 {
warn!(
"max_cleanup_wait_s and max_cleanup_backoff_ms are deprecated and ignored: each action \
attempt now gets its own directory, so retries never wait for a previous attempt's \
cleanup."
);
}

// Initialize directory cache if configured
let directory_cache = if let Some(cache_config) = &config.directory_cache {
Expand Down Expand Up @@ -705,8 +700,6 @@ pub async fn new_local_worker(
upload_action_result_config: &config.upload_action_result,
max_action_timeout,
max_upload_timeout,
max_cleanup_wait,
max_cleanup_backoff,
timeout_handled_externally: config.timeout_handled_externally,
directory_cache,
#[cfg(target_os = "linux")]
Expand Down
Loading
Loading