Add zstd at-rest CAS compression with byte-for-byte wire passthrough

.github/styles/config/vocabularies/TraceMachina/accept.txt

@@ -200,6 +200,7 @@ rules_scala
 [Tt]oolchainize
 [Tt]oolchainized
 [Vv]endored
+[Vv]alidator
 dev_dependency
 Fumadocs
 Turborepo
@@ -318,3 +319,24 @@ subtree
 hardlink
 multiplicatively
 SELinux
+[Pp]assthrough
+[Rr]ecompression
+[Rr]ecompressions
+[Dd]ecodable
+libzstd
+windowLog
+[Ss]eekable
+[Tt]ranscodes
+[Ii]nterop
+[Dd]esynchronization
+max_recompression_size
+EXDEV
+[Oo]bservability
+[Uu]nguessable
+fsync
+max_concurrent_staged_uploads
+max_concurrent_identity_ops
+max_inline_commit_size
+stage_timeout_s
+commit_timeout_s
+compressed_upload_idle_timeout_s

deployment-examples/docker-compose/README.md

@@ -121,6 +121,7 @@ services:
 ### Single Worker Setup
 - [`docker-compose.yml`](./docker-compose.yml) - Docker Compose file for single worker deployment
 - [`local-storage-cas.json5`](./local-storage-cas.json5) - Local storage CAS configuration for single worker
+- [`local-storage-cas-zstd.json5`](./local-storage-cas-zstd.json5) - Same, but the CAS selects `compression_algorithm.zstd` to keep blobs as zstd at rest for byte-for-byte `--remote_cache_compression` passthrough (see the file's header comment for the dedicated-namespace and placement rules)
 - [`scheduler.json5`](./scheduler.json5) - Scheduler configuration for single worker deployment
 - [`worker.json5`](./worker.json5) - Worker configuration for single worker deployment
 

deployment-examples/docker-compose/local-storage-cas-zstd.json5

@@ -0,0 +1,242 @@
+// Variant of `local-storage-cas.json5` that stores CAS blobs as zstd streams
+// at rest via the `zstd` algorithm of `compression`, instead of raw bytes
+// behind `compression` with its `lz4` algorithm.
+//
+// The payoff over plain `compression`: a Bazel client built with
+// `--remote_cache_compression` (enabled below via `remote_cache_compression`
+// on the `capabilities` service) gets served the stored zstd bytes
+// byte-for-byte on a cache hit — no decompress-then-recompress round trip at
+// the gRPC boundary. Identity clients (no `--remote_cache_compression`) are
+// unaffected: they still get plain decompressed bytes.
+//
+// Read this whole comment block before adapting this file. `compression` with
+// the `zstd` algorithm has deployment constraints that are not optional
+// footnotes; getting them wrong
+// either silently corrupts a mixed namespace or forces a cache flush later.
+// Full explanation:
+// https://github.com/TraceMachina/nativelink/blob/main/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx
+//
+// 1. DEDICATED, EMPTY NAMESPACE ONLY. The `compression.backend` below,
+//    when `compression_algorithm.zstd` is selected,
+//    (`CAS_ZSTD_CONTENT_STORE`, and the paths it writes) must never be shared
+//    with a store, instance, or NativeLink version that reads or writes those
+//    same keys as raw bytes. The digest key is identical either way, raw CAS
+//    values are unstructured (a raw blob can legitimately start with the
+//    zstd magic number), so there is no safe way to sniff which encoding an
+//    entry is in. That's why this file uses brand-new `*-cas-zstd` paths
+//    instead of the `content_path-cas` paths from `local-storage-cas.json5` —
+//    reusing those would mix raw and zstd values under one namespace.
+//
+// 2. NO IN-PLACE MIGRATION. Selecting `compression_algorithm.zstd` for an existing
+//    populated CAS, or turning it back off, both require a cache flush (or
+//    cutting over to a new namespace as done here) — not an in-place
+//    conversion. The same applies to a rolling/mixed-version deploy against
+//    one shared namespace: unsupported, because some replicas would write
+//    raw and others zstd to the same keys.
+//
+// 3. PLACEMENT: the `compression` store with `compression_algorithm.zstd` must
+//    be the store an instance/service points at directly for the passthrough
+//    optimization above to apply. Wrappers such
+//    as `fast_slow`, `dedup`, `existence_cache`, `cache_metrics`, `shard`,
+//    `ref_store`, and `size_partitioning` are fine *inside* it (they then
+//    operate on the physical zstd stream). A wrapper placed *outside*
+//    the zstd compression store (e.g. `verify` around it) is still correct,
+//    but forces that wrapper to decode the stream, which disables passthrough at that
+//    boundary.
+//
+// 4. CAS-ONLY. `compression_algorithm.zstd` only supports digest-keyed CAS entries; never
+//    point an AC store at it (`AC_MAIN_STORE` below stays plain `filesystem`).
+{
+  stores: [
+    {
+      name: "CAS_MAIN_STORE",
+      compression: {
+        compression_algorithm: {
+          zstd: {
+            // Staging directory for upload validation. MUST be on the same
+            // filesystem as the `filesystem` backend's `content_path` below:
+            // that backend commits a staged upload with rename(2), which fails
+            // with EXDEV across filesystems. Kept on the same
+            // `/root/.cache/nativelink` tree here for exactly that reason.
+            temp_path: "/root/.cache/nativelink/tmp_path-cas-zstd-stage",
+
+            // Reject a compressed upload over this many wire bytes with
+            // RESOURCE_EXHAUSTED rather than staging it.
+            max_compressed_upload_size: "512MiB",
+
+            // How many uploads may hold a staged file at once. Sizes worst-case
+            // temp disk usage together with max_compressed_upload_size:
+            //   max_concurrent_staged_uploads * max_compressed_upload_size
+            // Recompression adds no staging file (it reuses the same
+            // descriptor), but does hold up to max_recompression_size in memory.
+            // Monitor `temp_path` against the disk bound.
+            max_concurrent_staged_uploads: 4,
+
+            // How many uncompressed (identity) reads and writes to admit at
+            // once. Identity clients — anything without
+            // --remote_cache_compression — are encoded/decoded on a blocking
+            // thread held for the whole transfer, so this bound keeps a flood of
+            // slow identity clients from starving the process-wide blocking pool
+            // that filesystem I/O also uses.
+            max_concurrent_identity_ops: 256,
+
+            // Level used to encode uploads this store compresses itself, and to
+            // re-encode incoming compressed uploads when
+            // max_recompression_size > 0. Omit for the default of 3.
+            //
+            // Valid range is 1..=19 — enforced at startup. This is *not* the
+            // full zstd range (up to 22): standard levels 1-19 cap the encoded
+            // frame's window size at windowLog <= 23 (<= 8MiB), which every
+            // Bazel/zstd-jni client can decode with plain libzstd defaults.
+            // Bazel's decoder never calls setLongMax, so its hard ceiling is
+            // windowLog <= 27 (128MiB) - 19 keeps a large margin under that and
+            // is what caps the field, not a smaller "safe zstd" limit. Long-
+            // distance matching and dictionaries are never enabled by this
+            // store.
+            //
+            // 9 is a general-purpose default here: it costs a fraction of level
+            // 19's CPU per byte. Raise it toward 19 for an archival cache whose
+            // blobs are read far more often than written.
+            compression_level: 9,
+
+            // Recompression is only attempted for uploads whose *decoded* size
+            // is within this bound, and the smaller of the original and
+            // re-encoded stream is kept. 0 disables recompression entirely (an
+            // incoming compressed stream is always stored as-is), while still
+            // letting `compression_level` pick the level used for brand-new,
+            // non-precompressed uploads. A positive value requires
+            // `compression_level`; without it, startup fails rather than
+            // silently doing nothing.
+            max_recompression_size: "64MiB",
+
+            // Concurrent recompressions admitted by this store instance.
+            // Recompression is best-effort: an upload that finds every slot busy
+            // commits its original stream rather than queueing, so a small pool
+            // here throttles recompression without throttling uploads.
+            max_concurrent_recompressions: 1,
+
+            // Compressed uploads at or below this size are validated and
+            // committed straight from memory, with no staging file and no
+            // fsync. BatchUpdateBlobs payloads are small and numerous, so a
+            // per-blob disk round trip would dominate their cost.
+            max_inline_commit_size: "4MiB",
+
+            // Total time one upload may spend being validated and staged,
+            // measured from admission to a staging slot. Unlike the ByteStream
+            // per-message idle timeout, continuous slow progress does NOT reset
+            // it. On expiry the blocking validator retains its slot and cleanup
+            // guard until its input closes; ByteStream closes that input when
+            // the store fails. Size this against max_compressed_upload_size and
+            // the slowest upload bandwidth worth serving.
+            stage_timeout_s: 600,
+
+            // Time budget for a staged upload's optional recompression plus the
+            // inner-store commit, after the client stream is done. Bounds a
+            // stalled backend holding a staging slot.
+            commit_timeout_s: 300,
+          },
+        },
+
+        // MUST be a brand-new/empty namespace — see note 1 above. Everything
+        // under `backend` here is a complete, dedicated CAS stack; nothing
+        // else in this file (or any other deployment) may read or write it
+        // as raw bytes.
+        backend: {
+          fast_slow: {
+            fast: {
+              memory: {
+                eviction_policy: {
+                  // 500mb. Absorbs hot re-reads without touching disk.
+                  max_bytes: 500000000,
+                },
+              },
+            },
+            slow: {
+              filesystem: {
+                content_path: "/root/.cache/nativelink/content_path-cas-zstd",
+                temp_path: "/root/.cache/nativelink/tmp_path-cas-zstd",
+                eviction_policy: {
+                  // 10gb.
+                  max_bytes: 10000000000,
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+    {
+      // Holds blob-to-chunks layouts for the SplitBlob/SpliceBlob RPCs used
+      // by Bazel's --experimental_remote_cache_chunking. Must not verify
+      // digests and must not be the same store as the CAS.
+      name: "CHUNK_INDEX_STORE",
+      filesystem: {
+        content_path: "/root/.cache/nativelink/content_path-chunk-index",
+        temp_path: "/root/.cache/nativelink/tmp_path-chunk-index",
+        eviction_policy: {
+          // 100mb.
+          max_bytes: 100000000,
+        },
+      },
+    },
+    {
+      // Plain filesystem store: zstd compression is CAS-only (note 4 above), so
+      // the AC never goes anywhere near it.
+      name: "AC_MAIN_STORE",
+      filesystem: {
+        content_path: "/root/.cache/nativelink/content_path-ac",
+        temp_path: "/root/.cache/nativelink/tmp_path-ac",
+        eviction_policy: {
+          // 500mb.
+          max_bytes: 500000000,
+        },
+      },
+    },
+  ],
+  servers: [
+    {
+      listener: {
+        http: {
+          socket_address: "0.0.0.0:50051",
+        },
+      },
+      services: {
+        cas: [
+          {
+            cas_store: "CAS_MAIN_STORE",
+
+            // Optional: enables content-defined chunking
+            // (SplitBlob/SpliceBlob) for Bazel clients running with
+            // --experimental_remote_cache_chunking.
+            experimental_chunking: {
+              index_store: "CHUNK_INDEX_STORE",
+            },
+          },
+        ],
+        ac: [
+          {
+            ac_store: "AC_MAIN_STORE",
+          },
+        ],
+        capabilities: [
+          {
+            // Advertises zstd to clients so ByteStream/CAS RPCs accept and
+            // serve compressed-blobs/zstd/... payloads. This is what makes
+            // the byte-for-byte passthrough in the zstd compression store reachable —
+            // without it, every client uses identity transfers and
+            // zstd compression still stores zstd at rest, but every read pays
+            // a decompress.
+            remote_cache_compression: true,
+          },
+        ],
+        bytestream: {
+          cas_stores: {
+            "": "CAS_MAIN_STORE",
+          },
+        },
+        fetch: {},
+        push: {},
+      },
+    },
+  ],
+}