Add a split durable-dir capture that hardlinks file contents

cmd/atelet/main.go

@@ -921,18 +921,25 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp
 }
 
 // narrowFullCaptureToData rewrites rec so a FULL capture uploads as a DATA
-// snapshot. Each sandbox class owns one branch: micro-VM durable data is a
-// self-contained tar that can be carved out of the full file set; gVisor's
+// snapshot. Each sandbox class owns one branch: micro-VM durable data is
+// self-contained and can be carved out of the full file set by name; gVisor's
 // full checkpoint is monolithic until split checkpoints land.
 func narrowFullCaptureToData(rec *sandboxAssetsRecord) error {
 	switch atev1alpha1.SandboxClass(rec.SandboxClass) {
 	case atev1alpha1.SandboxClassMicroVM, atev1alpha1.SandboxClassGvisor:
-		if !slices.Contains(rec.SnapshotFiles, ateompath.DurableDirTarFile) {
+		// Selected by predicate rather than by name: a micro-VM checkpoint may
+		// have written the durable dir as an index plus blobs instead of one
+		// tar. A gVisor checkpoint only ever writes the tar, which the
+		// predicate also matches.
+		durable := slices.DeleteFunc(slices.Clone(rec.SnapshotFiles), func(f string) bool {
+			return !ateompath.DurableDirSnapshotFile(f)
+		})
+		if len(durable) == 0 {
 			// No durable-dir volumes were attached at pause: this snapshot
 			// holds no data, and never will — not retryable.
-			return status.Errorf(codes.FailedPrecondition, "full %s capture has no %s; the actor has no durable data to upload as %s", rec.SandboxClass, ateompath.DurableDirTarFile, ateattr.SnapshotScopeData)
+			return status.Errorf(codes.FailedPrecondition, "full %s capture has no durable-dir files; the actor has no durable data to upload as %s", rec.SandboxClass, ateattr.SnapshotScopeData)
 		}
-		rec.SnapshotFiles = []string{ateompath.DurableDirTarFile}
+		rec.SnapshotFiles = durable
 		rec.Scope = ateattr.SnapshotScopeData
 		return nil
 

cmd/atelet/main_test.go

@@ -1727,6 +1727,55 @@ func TestUploadLocalCheckpointDir(t *testing.T) {
 		}
 	})
 
+	// The split arrangement spreads durable data over an index and one blob per
+	// file, so the carve has to select a set rather than a single name — and
+	// still leave the guest state behind.
+	t.Run("microvm full capture uploads the durable index and blobs as data", func(t *testing.T) {
+		store := &recordingObjectStorage{}
+		s := &AteomHerder{gcsClient: store}
+		dir := filepath.Join(t.TempDir(), "pause-snap-1")
+		rec := fullRec("microvm")
+		rec.SnapshotFiles = []string{
+			"config.json", "memory-ranges",
+			ateompath.DurableDirIndexFile,
+			ateompath.DurableDirBlobPrefix + "0000",
+			ateompath.DurableDirBlobPrefix + "0001",
+		}
+		writeLocalSnapshot(t, dir, rec, map[string]string{
+			"config.json": "cfg", "memory-ranges": "mem",
+			ateompath.DurableDirIndexFile:           "index",
+			ateompath.DurableDirBlobPrefix + "0000": "one",
+			ateompath.DurableDirBlobPrefix + "0001": "two",
+		})
+
+		req := validUploadPausedCheckpointRequest()
+		req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA
+		if _, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri); err != nil {
+			t.Fatalf("uploadLocalCheckpointDir: %v", err)
+		}
+		want := []string{
+			pausedSnapshotPath + "/durable-dir.blob-0000.zstd",
+			pausedSnapshotPath + "/durable-dir.blob-0001.zstd",
+			pausedSnapshotPath + "/durable-dir.index.tar.zstd",
+			pausedSnapshotPath + "/manifest.json",
+		}
+		if got := store.keys(); !slices.Equal(got, want) {
+			t.Errorf("uploaded objects = %v, want %v", got, want)
+		}
+		uploaded := remoteManifest(t, store)
+		if uploaded.Scope != ateattr.SnapshotScopeData {
+			t.Errorf("uploaded manifest scope = %q, want %q", uploaded.Scope, ateattr.SnapshotScopeData)
+		}
+		want = []string{
+			ateompath.DurableDirIndexFile,
+			ateompath.DurableDirBlobPrefix + "0000",
+			ateompath.DurableDirBlobPrefix + "0001",
+		}
+		if !slices.Equal(uploaded.SnapshotFiles, want) {
+			t.Errorf("uploaded manifest files = %v, want %v", uploaded.SnapshotFiles, want)
+		}
+	})
+
 	t.Run("gvisor full capture without durable tar has no data", func(t *testing.T) {
 		s := &AteomHerder{gcsClient: &recordingObjectStorage{}}
 		dir := filepath.Join(t.TempDir(), "pause-snap-1")

cmd/ateom-microvm/checkpoint.go

@@ -130,7 +130,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 	}
 
 	// Capture the snapshot's pieces CONCURRENTLY: the CH snapshot, the
-	// durable-dir tar, and the rootfs upper tar read independent data from a
+	// durable-dir capture, and the rootfs upper tar read independent data from a
 	// quiesced guest and write distinct files into checkpointDir, so the paused
 	// window costs the slowest of them rather than their sum (the tars scale
 	// with the actor's data; suspend latency is the metric that matters).
@@ -140,8 +140,8 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 	//     since nothing will reattach to the frozen virtio-fs lower: at restore
 	//     the actor cold-boots from the OCI image (or, under an OnGolden data
 	//     resume policy, is combined with the golden snapshot's guest state).
-	//   - Durable-dir tar (any scope, when declared): host-backed, so pausing
-	//     the write-through share makes the tar coherent.
+	//   - Durable-dir capture (any scope, when declared): host-backed, so
+	//     pausing the write-through share makes it coherent.
 	//   - Rootfs upper tar (Full only): host-backed like the durable volumes —
 	//     the memory snapshot does not carry rootfs writes. Under Data the
 	//     workload cold-starts on restore, discarding rootfs state.
@@ -157,7 +157,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 	if durable {
 		g.Go(func() error {
 			t := time.Now()
-			if err := tarDurableVolumes(gctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil {
+			if err := captureDurableVolumes(gctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil {
 				return err
 			}
 			dDurable = time.Since(t)
@@ -180,7 +180,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 
 	// Report exactly the files we wrote so atelet ships precisely this snapshot: for
 	// Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus
-	// any durable-dir tar; for Data, that tar alone.
+	// any durable-dir files; for Data, those alone.
 	snapshotFiles, err := listFiles(checkpointDir)
 	if err != nil {
 		return nil, fmt.Errorf("while listing snapshot files: %w", err)
@@ -201,9 +201,11 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 	slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles),
 		slog.String("scope", scope.String()), slog.Duration("pause", dPause),
 		slog.Duration("snapshot", dSnapshot),
-		// The tars run while the guest is paused, CONCURRENTLY with the CH
+		// The captures run while the guest is paused, CONCURRENTLY with the CH
 		// snapshot: the paused window costs max(snapshot, durable_dir,
-		// rootfs_upper), and the tar durations scale with the actor's data.
+		// rootfs_upper). rootfs_upper scales with the actor's data, and so does
+		// durable_dir unless the split arrangement is on, where it scales with
+		// the file count instead.
 		slog.Duration("durable_dir", dDurable), slog.Duration("rootfs_upper", dUpper),
 		slog.Duration("teardown", dTeardown))
 	return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil

cmd/ateom-microvm/durable.go

@@ -29,14 +29,29 @@ package main
 // virtio-fs share at SharedDir(actorUID)/durable, where each container's bind
 // is attached.
 //
-// Snapshots carry the contents as a tar of the whole per-actor directory, so
-// every volume rides along and the layout is reproduced verbatim on restore.
+// Snapshots carry the contents of the whole per-actor directory, so every
+// volume rides along and the layout is reproduced verbatim on restore.
 // virtiofsd serves the share write-through (no --writeback), so once the guest
 // is paused every completed guest write is already visible on the host and the
-// tar is complete.
+// capture is complete.
+//
+// There are two arrangements for that capture, chosen by ATEOM_DURABLE_BACKEND:
+//
+//   - tar (the default): one archive of the whole directory. Sealing it reads
+//     and rewrites every byte the actor holds, on the paused critical path.
+//   - files: a metadata-only index tar plus one blob per non-empty regular
+//     file, hard-linked out of the directory rather than copied. Sealing costs
+//     one link per file instead of a copy of the tree, so the paused window
+//     stops scaling with the actor's data. The price is a snapshot of many
+//     objects rather than one, which a directory of many small files pays for
+//     on upload.
+//
+// A restore dispatches on what the snapshot actually contains, not on the
+// variable, so either arrangement reads back under either setting.
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -48,12 +63,30 @@ import (
 	"github.com/agent-substrate/substrate/internal/tarutil"
 )
 
-// durableTarFile is the snapshot file holding the tar of the actor's durable-dir
-// volumes. Its entries are <volumeName>/... relative to
-// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout.
-// The name is shared with atelet, which uses it to carve durable data out of a
-// FULL snapshot's file set when uploading a paused checkpoint as DATA.
-const durableTarFile = ateompath.DurableDirTarFile
+// The snapshot files holding the actor's durable-dir volumes, in whichever
+// arrangement wrote them. Entries are <volumeName>/... relative to
+// ateompath.DurableDirVolumeMountsDir, so a restore reproduces the same layout.
+// The names are shared with atelet, which uses them to carve durable data out
+// of a FULL snapshot's file set when uploading a paused checkpoint as DATA.
+const (
+	durableTarFile    = ateompath.DurableDirTarFile
+	durableIndexFile  = ateompath.DurableDirIndexFile
+	durableBlobPrefix = ateompath.DurableDirBlobPrefix
+)
+
+// durableBackendEnvVar selects the arrangement a checkpoint writes. Only
+// durableBackendFiles is recognized; anything else, including unset, keeps the
+// tar.
+const (
+	durableBackendEnvVar = "ATEOM_DURABLE_BACKEND"
+	durableBackendFiles  = "files"
+)
+
+// splitDurableCapture reports whether checkpoints should write the split
+// arrangement.
+func splitDurableCapture() bool {
+	return os.Getenv(durableBackendEnvVar) == durableBackendFiles
+}
 
 // hasDurableVolumes reports whether any container mounts a durable-dir volume.
 func hasDurableVolumes(containers []*ateompb.Container) bool {
@@ -78,28 +111,60 @@ func (s *AteomService) stageDurableVolumes(ctx context.Context, actorUID string)
 	return nil
 }
 
-// tarDurableVolumes archives the actor's durable-dir volumes (dir) into the
+// captureDurableVolumes writes the actor's durable-dir volumes (dir) into the
 // checkpoint directory. The caller must have paused the guest first: virtiofsd is
 // write-through, so a completed guest write is on the host by then, but a
 // running guest could still add more after the walk.
 //
 // Sockets the workload left behind are skipped rather than archived (tarutil
 // logs them); they hold no data and the workload recreates them on start.
-func tarDurableVolumes(ctx context.Context, dir, checkpointDir string) error {
-	if err := tarutil.Create(ctx, filepath.Join(checkpointDir, durableTarFile), dir); err != nil {
+//
+// The split arrangement hardlinks file contents into the checkpoint directory,
+// which means dir must not be written again afterwards. CheckpointWorkload
+// guarantees that: the guest stays paused until terminateWorkload tears the
+// sandbox down, and atelet resets the actor's directories after the RPC
+// returns.
+func captureDurableVolumes(ctx context.Context, dir, checkpointDir string) error {
+	var err error
+	if splitDurableCapture() {
+		_, err = tarutil.CreateSplit(ctx, filepath.Join(checkpointDir, durableIndexFile), dir, checkpointDir, durableBlobPrefix)
+	} else {
+		err = tarutil.Create(ctx, filepath.Join(checkpointDir, durableTarFile), dir)
+	}
+	if err != nil {
 		return fmt.Errorf("while archiving durable-dir volumes from %q: %w", dir, err)
 	}
 	return nil
 }
 
-// untarDurableVolumes restores the durable-dir volumes from a snapshot into the
+// restoreDurableVolumes restores the durable-dir volumes from a snapshot into the
 // actor's host directory (dir, which atelet has already created, empty). It must
 // run before the durable share's virtiofsd starts, so the guest never observes
 // the directory mid-restore.
-func untarDurableVolumes(dir, snapshotDir string) error {
+//
+// Which arrangement the snapshot is in is read off the snapshot itself: a node
+// configured for one has to be able to restore an actor captured under the
+// other, and the snapshot outlives whatever ateom wrote it.
+//
+// The split arrangement hands the guest the staged blobs themselves, hard-linked
+// rather than copied, so the guest's writes land on snapshotDir's inodes. That
+// is sound because snapshotDir belongs to this activation alone: atelet stages
+// it by copying out of the retained checkpoint — so a write can never reach the
+// checkpoint every later restore reads — nothing reads it once this call
+// returns, and resetActorDirs deletes it together with dir.
+func restoreDurableVolumes(dir, snapshotDir string) error {
 	if err := os.MkdirAll(dir, 0o755); err != nil {
 		return fmt.Errorf("while creating durable-dir volumes dir %q: %w", dir, err)
 	}
+	index := filepath.Join(snapshotDir, durableIndexFile)
+	if _, err := os.Stat(index); err == nil {
+		if err := tarutil.ExtractSplit(index, snapshotDir, dir); err != nil {
+			return fmt.Errorf("while restoring durable-dir volumes into %q: %w", dir, err)
+		}
+		return nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("while checking for durable-dir index %q: %w", index, err)
+	}
 	if err := tarutil.Extract(filepath.Join(snapshotDir, durableTarFile), dir); err != nil {
 		return fmt.Errorf("while restoring durable-dir volumes into %q: %w", dir, err)
 	}