atelet: serve golden snapshot restores from the node-local file cache

cmd/ateapi/internal/controlapi/functionaltest/actor_test.go

@@ -1910,12 +1910,120 @@ func TestResumeActorPassesLiteralEnv(t *testing.T) {
 	}
 }
 
-// TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available.
-// Workflow:
-// 1. Creates a mock ActorTemplate.
-// 2. Creates an actor.
-// 3. Calls ResumeActor RPC without creating any workers.
-// 4. Verifies that ResumeActor fails with FailedPrecondition status.
+// createGoldenDataTemplate creates "tmpl1" like createTemplate, but with
+// onCommit DATA and onResume.fromData GOLDEN, so a resumed-after-suspend
+// actor takes the DATA_ON_GOLDEN path: its data snapshot combined with the
+// template's golden.
+func createGoldenDataTemplate(t *testing.T, tc *testContext, ns string) *ateapipb.ActorTemplate {
+	t.Helper()
+	ensureDefaultGvisorSandboxConfig(t, tc)
+	createWorkerPool(t, tc, ns, "pool1", map[string]string{poolLabelKey: ns})
+
+	created, err := tc.client.CreateActorTemplate(context.Background(), &ateapipb.CreateActorTemplateRequest{
+		ActorTemplate: &ateapipb.ActorTemplate{
+			Metadata: &ateapipb.ResourceMetadata{
+				Atespace: testAtespace,
+				Name:     "tmpl1",
+			},
+			SnapshotsConfig: &ateapipb.SnapshotsConfig{
+				StorageLocation: testStorageLocation,
+				OnPause:         ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL,
+				OnCommit:        ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
+				OnResume:        &ateapipb.OnResumeConfig{FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN},
+			},
+			SandboxConfig: &ateapipb.SandboxConfig{
+				SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR,
+				ConfigName:   "gvisor-default",
+			},
+			Containers: []*ateapipb.Container{{
+				Name:    "main",
+				Image:   "main@sha256:abc",
+				Command: []string{"/main"},
+			}},
+			WorkerSelector: &ateapipb.Selector{
+				MatchLabels: map[string]string{poolLabelKey: ns},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("failed to create actor template: %v", err)
+	}
+	updated, err := tc.persistence.UpdateActorTemplate(context.Background(),
+		resources.ActorTemplateRefFromActorTemplate(created), store.PreconditionFrom(created),
+		func(dbTemplate *ateapipb.ActorTemplate) error {
+			dbTemplate.Status = &ateapipb.ActorTemplateStatus{
+				GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{
+					GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: goldenSnapshotURI(t), ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL},
+				},
+			}
+			return nil
+		})
+	if err != nil {
+		t.Fatalf("failed to record the template's golden snapshot: %v", err)
+	}
+	return updated
+}
+
+// TestResumeActor_GoldenDataResumeSetsBaseConfig drives the DATA_ON_GOLDEN
+// resume end to end and pins the wire request's base snapshot fields: while
+// the golden_snapshot_uri -> base_config transition lasts, ateapi sets both
+// and they must agree, so ateapi and atelet can roll in either order.
+func TestResumeActor_GoldenDataResumeSetsBaseConfig(t *testing.T) {
+	ns := namespaceForTest("ns-resume-golden-data")
+	tc := setupTest(t, ns)
+	defer tc.cleanup()
+
+	tmpl := createGoldenDataTemplate(t, tc, ns)
+	workerName := createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1")
+
+	const name = "id1"
+	actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: name}
+	if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{
+		Metadata:      &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name},
+		ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"},
+	}}); err != nil {
+		t.Fatalf("CreateActor failed: %v", err)
+	}
+
+	// First resume runs fresh from the golden; the suspend then commits a
+	// DATA snapshot per onCommit.
+	if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("ResumeActor (first) failed: %v", err)
+	}
+	suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: actorRef})
+	if err != nil {
+		t.Fatalf("SuspendActor failed: %v", err)
+	}
+	waitForWorkerAvailable(t, tc, workerName)
+	actorSnapshotURI := suspended.GetActor().GetStatus().GetExternalSnapshot().GetSnapshotUri()
+	if actorSnapshotURI == "" {
+		t.Fatal("SuspendActor recorded no external snapshot")
+	}
+
+	// Second resume: the actor's DATA snapshot rides on the template's
+	// golden.
+	if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil {
+		t.Fatalf("ResumeActor (second) failed: %v", err)
+	}
+	restoreReq := tc.fakeAtelet.lastRestoreRequest()
+	if restoreReq == nil {
+		t.Fatal("second resume sent no Restore request to atelet")
+	}
+	if got := restoreReq.GetScope(); got != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN {
+		t.Fatalf("restore scope = %v, want SNAPSHOT_SCOPE_DATA_ON_GOLDEN", got)
+	}
+	if got := restoreReq.GetExternalConfig().GetSnapshotUri(); got != actorSnapshotURI {
+		t.Errorf("restore config snapshot uri = %q, want the actor's data snapshot %q", got, actorSnapshotURI)
+	}
+	golden := tmpl.GetStatus().GetGoldenSnapshotStatus().GetGoldenSnapshot().GetSnapshotUri()
+	if got := restoreReq.GetBaseConfig().GetSnapshotUri(); got != golden {
+		t.Errorf("restore base_config uri = %q, want the template's golden %q", got, golden)
+	}
+	if got := restoreReq.GetGoldenSnapshotUri(); got != golden {
+		t.Errorf("restore golden_snapshot_uri = %q, want %q (transitional dual-write must match base_config)", got, golden)
+	}
+}
+
 // TestResumeActor_NoWorkers tests that resuming an actor fails when no free workers are available.
 // Workflow:
 // 1. Creates a mock ActorTemplate.
@@ -2476,6 +2584,9 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) {
 			if got := restoreReq.GetGoldenSnapshotUri(); got != "" {
 				t.Errorf("restore request to atelet had golden snapshot uri = %q, want empty", got)
 			}
+			if restoreReq.GetBaseConfig() != nil {
+				t.Errorf("restore request to atelet had base_config = %v, want unset", restoreReq.GetBaseConfig())
+			}
 		})
 	}
 }

cmd/ateapi/internal/controlapi/workflow_resume.go

@@ -700,6 +700,10 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
 			req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA
 		case !src.GoldenSnapshotURI.IsZero():
 			req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN
+			// Transitional dual-write: base_config supersedes
+			// golden_snapshot_uri, but an atelet from before it reads only
+			// the old field. Dropped once both components have rolled.
+			req.BaseConfig = &ateletpb.ExternalRestoreConfiguration{SnapshotUri: src.GoldenSnapshotURI.String()}
 			req.GoldenSnapshotUri = src.GoldenSnapshotURI.String()
 		default:
 			req.Scope = actorSnapshotContentScopeToAtelet(actorTemplate.GetSnapshotsConfig().GetOnPause())
@@ -718,11 +722,16 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
 		}
 		var scope ateletpb.SnapshotScope
 		var goldenSnapshotURI string
+		var baseConfig *ateletpb.ExternalRestoreConfiguration
 		switch {
 		case src.TemplateReplaced:
 			scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA
 		case !src.GoldenSnapshotURI.IsZero():
 			scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN
+			// Transitional dual-write: base_config supersedes
+			// golden_snapshot_uri, but an atelet from before it reads only
+			// the old field. Dropped once both components have rolled.
+			baseConfig = &ateletpb.ExternalRestoreConfiguration{SnapshotUri: src.GoldenSnapshotURI.String()}
 			goldenSnapshotURI = src.GoldenSnapshotURI.String()
 		default:
 			scope = actorSnapshotContentScopeToAtelet(src.Scope)
@@ -737,12 +746,13 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou
 			Spec:                  workloadSpec,
 			Type:                  ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL,
 			Config: &ateletpb.RestoreRequest_ExternalConfig{
-				ExternalConfig: &ateletpb.ExternalCheckpointConfiguration{
+				ExternalConfig: &ateletpb.ExternalRestoreConfiguration{
 					SnapshotUri: src.SnapshotURI.String(),
 				},
 			},
 			Scope: scope,
-			// Empty unless this is a Golden data resume.
+			// Both empty unless this is a Golden data resume.
+			BaseConfig:        baseConfig,
 			GoldenSnapshotUri: goldenSnapshotURI,
 			ActorUid:          actor.GetMetadata().Uid,
 			EgressGateway:     egressGateway,

cmd/atelet/goldencache.go

@@ -0,0 +1,169 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+// The golden snapshot file cache.
+//
+// A golden snapshot's files are immutable once published (a changed golden
+// gets a new URI), yet every restore that needs them — fresh-from-golden
+// starts and DATA_ON_GOLDEN resumes — downloads them again into its own
+// per-actor dir. This cache (a filecache.Store) makes those files a
+// node-level resource: concurrent restores of one golden share a single
+// download, and later restores hard-link the cached copy instead of
+// fetching it.
+//
+// This file owns the cache's lifecycle (flags, validation, opening with the
+// startup debris sweep) and the restore path's cached reads. The eviction
+// loop is wired separately.
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"syscall"
+	"time"
+
+	"github.com/agent-substrate/substrate/cmd/atelet/internal/ategcs"
+	"github.com/agent-substrate/substrate/cmd/atelet/internal/filecache"
+	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/resources"
+	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
+	"github.com/spf13/pflag"
+)
+
+var (
+	goldenCacheDir    = pflag.String("golden-cache-dir", ateompath.GoldenCacheDir, "Directory for the node-local golden snapshot file cache. Empty disables caching (every restore downloads its golden files). Must be on the same filesystem mount as the actor state dirs: cache hits are served as hard links into the per-actor restore dirs.")
+	goldenCacheMinAge = pflag.Duration("golden-cache-min-age", 10*time.Minute, "Cached golden files younger than this are never evicted, protecting files fetched but not yet linked into a restore dir.")
+)
+
+func validateGoldenCacheFlags() error {
+	if *goldenCacheMinAge < 0 {
+		// A negative min-age inverts the veto (the cutoff lands in the
+		// future), making just-fetched files evictable mid-restore.
+		return fmt.Errorf("--golden-cache-min-age %v must be >= 0", *goldenCacheMinAge)
+	}
+	if *goldenCacheDir != "" && !ateompath.UnderBasePath(*goldenCacheDir) {
+		slog.Warn("Golden cache dir is outside the ateom base path; hits cannot be hard-linked into restore dirs across mounts, so every hit degrades to a copy",
+			slog.String("golden_cache_dir", *goldenCacheDir),
+			slog.String("actors_dir", ateompath.ActorsDir))
+	}
+	return nil
+}
+
+// openGoldenCache opens the golden snapshot cache rooted at dir and clears
+// crash debris before the store serves any restore. An empty dir disables
+// caching: the returned store is nil and restores download their golden
+// files directly.
+func openGoldenCache(ctx context.Context, dir string, minAge time.Duration) (*filecache.Store, error) {
+	if dir == "" {
+		slog.InfoContext(ctx, "Golden snapshot cache disabled; every restore downloads its golden files")
+		return nil, nil
+	}
+	store, err := filecache.New(dir, filecache.WithMinAge(minAge))
+	if err != nil {
+		return nil, fmt.Errorf("while opening golden cache at %s: %w", dir, err)
+	}
+	stats, err := store.SweepDebris(ctx)
+	if err != nil {
+		// Leftover debris wastes space but affects no lookup, so the store
+		// is fully usable: log and carry on rather than failing startup.
+		slog.WarnContext(ctx, "Golden snapshot cache debris sweep incomplete", slog.Any("err", err))
+	}
+	slog.InfoContext(ctx, "Golden snapshot cache open",
+		slog.String("dir", dir),
+		slog.Int("tmp_removed", stats.TmpRemoved),
+		slog.Int("retired_removed", stats.RetiredRemoved))
+	return store, nil
+}
+
+// isGoldenSnapshotURI reports whether snapshotURI names a golden snapshot:
+// one owned by an actor in the reserved golden atespace. A fresh-from-golden
+// start arrives as an ordinary external FULL restore whose snapshot URI is
+// the template's golden, so this is how the download path recognizes an
+// immutable, cache-safe source. Per-actor snapshot URIs are never cached:
+// each is used by one actor and deleted on its next suspend, so caching
+// them buys nothing.
+func isGoldenSnapshotURI(snapshotURI string) bool {
+	uri, err := resources.ParseSnapshotURI(snapshotURI)
+	return err == nil && uri.Atespace() == resources.GoldenActorAtespace
+}
+
+// goldenCacheMode is how a restore may materialize golden files from the
+// cache. The mode exists because the fast path — a hard link — shares the
+// cached inode with the consumer, which is only safe when the consumer never
+// writes the staged file in place.
+type goldenCacheMode int
+
+const (
+	// cacheModeOff downloads fresh, bypassing the cache.
+	cacheModeOff goldenCacheMode = iota
+	// cacheModeLink hard-links the cached copy: zero-cost hits, but the
+	// consumer must treat the staged file as read-only.
+	cacheModeLink
+	// cacheModeCopy stages a private, hole-preserving copy: costlier per hit
+	// than a link, but the consumer owns the inode and may mutate it in
+	// place, and a copy can cross mounts.
+	cacheModeCopy
+)
+
+// goldenCacheModeFor returns how this sandbox class's restores may use the
+// golden cache. ateom-gvisor consumes restore-state strictly read-only, so
+// it gets hard links. ateom-microvm rewrites config.json in place at restore
+// and merges checkpoint deltas into memory-ranges' inode at suspend — either
+// would corrupt a shared inode — so it gets private copies: still one
+// download per golden per node, and its mutations stay its own.
+func goldenCacheModeFor(sandboxClass string) goldenCacheMode {
+	switch atev1alpha1.SandboxClass(sandboxClass) {
+	case atev1alpha1.SandboxClassGvisor:
+		return cacheModeLink
+	case atev1alpha1.SandboxClassMicroVM:
+		return cacheModeCopy
+	default:
+		return cacheModeOff
+	}
+}
+
+// fetchSnapshotObject stages one snapshot object at local — through the
+// golden cache per mode when the cache is enabled, directly from object
+// storage otherwise. A cacheModeLink hit is a hard link, so local stays
+// valid regardless of later eviction; a cacheModeCopy hit is a private
+// copy. Fetch errors pass through the cache wrapped, keeping ateerrors
+// classification intact.
+func (s *AteomHerder) fetchSnapshotObject(ctx context.Context, objectURI, local string, mode goldenCacheMode) error {
+	if mode == cacheModeOff || s.goldenCache == nil {
+		return ategcs.FetchLocalFileFromGCSWithZstd(ctx, s.gcsClient, objectURI, local)
+	}
+	key := filecache.URIKey("gcs-zstd", objectURI)
+	fetch := func(ctx context.Context, dst string) error {
+		return ategcs.FetchLocalFileFromGCSWithZstd(ctx, s.gcsClient, objectURI, dst)
+	}
+	if mode == cacheModeCopy {
+		return s.goldenCache.GetFileCopyTo(ctx, key, local, fetch)
+	}
+	err := s.goldenCache.GetFileTo(ctx, key, local, fetch)
+	if err == nil || !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+	// The cache sits on a different mount than the restore dir, so link-out
+	// cannot work — but a copy can, and the failed link's flight already
+	// published the entry, so this is a local read rather than a second
+	// download. The warning points at the misconfiguration (a cache under
+	// --golden-cache-dir off the base-path mount serves every hit the slow
+	// way).
+	slog.WarnContext(ctx, "Golden cache is on a different filesystem than the restore dir; serving a copy instead of a hard link",
+		slog.String("object", objectURI))
+	return s.goldenCache.GetFileCopyTo(ctx, key, local, fetch)
+}