From 8b36b583e71fb5e275919cdd5b168ce72f91fa34 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 4 Sep 2026 14:10:41 -0700 Subject: [PATCH 1/2] ateapi: resolve the resume boot source and wire scope once in loadActorForResume The restore step derived the wire scope twice, once per restore branch, each with a different fallback (the template's onPause scope for a local pause checkpoint, the durable snapshot's captured scope otherwise), from the raw inputs resumeSnapshotSource carried (Scope, TemplateReplaced). Every input is already in hand when loadActorForResume resolves the boot source, so resolve everything there: the boot-source decision becomes an explicit first-match rule list (explicit boot / repointed / paused / no snapshot / own durable snapshot), the struct carries the resolved WireScope, and ensureAteletRestored only consumes it, attaching the golden URI only when the scope is DATA_ON_GOLDEN. The golden snapshot's validation and location parsing, previously duplicated between the no-snapshot golden fallback and the Golden data-resume policy, collapse into resolveGoldenSnapshot. GoldenSnapshotURI becomes GoldenForDataSnapshotURI: it only ever names the guest half of a data-only combined restore, never the boot snapshot itself. Two deliberate behavior changes ride along: - An explicit boot request now always restores data-only: it discards any captured guest state and carries the actor's durable data alone, where it previously restored a Full capture at Full scope (or rode the golden under the Golden policy). Boot is the operator's escape hatch to a clean guest, so it must not depend on stored guest state. - A paused actor whose template records an unusable golden snapshot no longer fails its resume: the local checkpoint restore never touches the golden fallback, which only applies to actors with no snapshot of their own. --- .../internal/controlapi/workflow_resume.go | 213 ++++++++++++------ .../controlapi/workflow_resume_test.go | 151 +++++++++++-- 2 files changed, 275 insertions(+), 89 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index c3b6fca08d..082e03903d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -42,15 +42,21 @@ type resumeSnapshotSource struct { // golden snapshot otherwise. Zero means cold boot from the spec (unless // the actor holds a local snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI - Scope ateapipb.SnapshotContentScope - // GoldenSnapshotURI is the storage location of the ActorTemplate's golden - // snapshot. Populated only when the template's onResume configuration - // selects the golden snapshot as the boot source for the pending restore: - // restore then combines the golden snapshot with the actor's data. - GoldenSnapshotURI resources.SnapshotURI - // TemplateReplaced is true when the snapshot's recorded template UID - // differs from the actor's current template. - TemplateReplaced bool + // GoldenForDataSnapshotURI is the storage location of the ActorTemplate's + // golden snapshot. Populated only when the template's onResume + // configuration selects the golden snapshot for a pending data-only + // restore: the restore then combines the golden snapshot with the + // actor's data. + GoldenForDataSnapshotURI resources.SnapshotURI + // WireScope is the scope of the restore operation, not of any stored + // snapshot: + // - DATA_ON_GOLDEN: the restore rides the golden snapshot. + // - DATA: the restore discards the snapshot's guest state and carries + // the actor's durable data alone. + // - FULL or unspecified: the restore uses a FULL snapshot. + // + // Meaningless for a cold boot: RunRequest carries no scope. + WireScope ateletpb.SnapshotScope } // restoreTelemetry labels the restore operation for the resume lifecycle @@ -154,7 +160,21 @@ func validateGoldenSnapshotScope(snapshot *ateapipb.ExternalSnapshot) error { } // loadActorForResume fetches the current actor record and its template, and -// resolves the boot source for the pending restore. +// resolves the boot source for the pending restore. The source is decided by +// the first matching rule: +// +// 1. An explicit boot request restores data-only from whatever snapshot +// exists (none: cold boot from the spec), never riding a golden. +// 2. A repointed actor's snapshot holds guest state from the replaced +// template, so the restore carries the actor's durable data alone. +// 3. A paused actor restores its local pause checkpoint at the template's +// onPause scope; a Data-scoped pause rides the golden when the onResume +// policy selects it. +// 4. An actor on its own template restores its snapshot at the captured +// scope; a Data-scoped capture rides the golden when the onResume +// policy selects it. +// 5. An actor with no snapshot restores the template's golden when one +// exists, and cold boots from the spec otherwise. func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef, boot bool) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, _ resumeSnapshotSource, err error) { ctx, done := stepSpan(ctx, "LoadActorForResume") defer func() { err = done(err) }() @@ -179,59 +199,117 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if err != nil { return nil, nil, src, err } - goldenSnapshotStatus := actorTemplate.GetStatus().GetGoldenSnapshotStatus() - if uri := actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); uri != "" { + external := actor.GetStatus().GetExternalSnapshot() + if uri := external.GetSnapshotUri(); uri != "" { if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err) } - src.Scope = actor.GetStatus().GetExternalSnapshot().GetContentScope() - // The Actor records the template its guest state was built on; a - // different UID on its current template means it was repointed since - // the capture. - builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid() - src.TemplateReplaced = builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid() - } else if goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri(); goldenURI != "" && !boot { - if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil { - return nil, nil, src, err - } - if src.SnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil { - return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err) - } - src.Scope = goldenSnapshotStatus.GetGoldenSnapshot().GetContentScope() - } - - // The template's onResume configuration selects the boot source for the - // pending restore. When it names the golden snapshot, resolve the golden - // snapshot's location so the restore can combine the golden's guest - // state with the actor's data. The pending - // restore is data-only when the actor is paused with a Data pause scope - // (the local snapshot takes precedence at restore), or when its durable - // snapshot holds Data. Valid Full snapshots restore from their own - // content and ignore the policy. - if actorTemplate.GetSnapshotsConfig().GetOnResume().GetFromData() == ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN { - dataOnly := false - if actor.GetStatus().GetLocalSnapshotInfo() != nil { - dataOnly = effectiveContentScope(actorTemplate.GetSnapshotsConfig().GetOnPause()) == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA - } else if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "" { - dataOnly = src.Scope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA - } - if dataOnly { - goldenURI := goldenSnapshotStatus.GetGoldenSnapshot().GetSnapshotUri() - if goldenURI == "" { - return nil, nil, src, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") - } - if err := validateGoldenSnapshotScope(goldenSnapshotStatus.GetGoldenSnapshot()); err != nil { - return nil, nil, src, err + } + + // Rule 1: an explicit boot request discards any captured guest state and + // carries the actor's durable data alone (none: cold boot from the spec). + if boot { + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + return actor, actorTemplate, src, nil + } + + onResumeGolden := actorTemplate.GetSnapshotsConfig().GetOnResume().GetFromData() == ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN + // Rule 2: An actor whose template was updated will cold boot. + builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid() + if external.GetSnapshotUri() != "" && builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid() { + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + return actor, actorTemplate, src, nil + } + + // Rule 3: a paused actor restores its local pause checkpoint, which the + // template's onPause scope describes. + if actor.GetStatus().GetLocalSnapshotInfo() != nil { + if actorTemplate.GetSnapshotsConfig().GetOnPause() == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { + if onResumeGolden { + if src.GoldenForDataSnapshotURI, err = resolveGoldenForDataResume(actorTemplate); err != nil { + return nil, nil, src, err + } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN + return actor, actorTemplate, src, nil } - if src.GoldenSnapshotURI, err = resources.ParseSnapshotURI(goldenURI); err != nil { - return nil, nil, src, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", goldenURI, err) + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + return actor, actorTemplate, src, nil + } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL + return actor, actorTemplate, src, nil + } + + // Rule 4: the actor restores its own durable snapshot at the captured + // scope; the onResume policy only governs data-only restores, so a valid + // Full (or legacy Unspecified) capture restores from its own content. + if external.GetSnapshotUri() != "" { + if external.GetContentScope() == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { + if onResumeGolden { + if src.GoldenForDataSnapshotURI, err = resolveGoldenForDataResume(actorTemplate); err != nil { + return nil, nil, src, err + } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN + return actor, actorTemplate, src, nil } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + return actor, actorTemplate, src, nil } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL + return actor, actorTemplate, src, nil } + // Rule 5: an actor with no snapshot of its own restores the template's + // golden snapshot as its own content when one exists, and cold boots + // from the spec otherwise (the wire scope is inert then: RunRequest + // carries no scope). + goldenURI, err := resolveGoldenSnapshot(actorTemplate) + if errors.Is(err, errNoGoldenSnapshot) { + return actor, actorTemplate, src, nil + } + if err != nil { + return nil, nil, src, err + } + src.SnapshotURI = goldenURI + // A golden snapshot is always FULL scope. + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL return actor, actorTemplate, src, nil } +// errNoGoldenSnapshot reports that the ActorTemplate records no golden +// snapshot. Callers decide what that means for their resume path: a gRPC +// status naming the path that needed the golden, or a fallback source. +var errNoGoldenSnapshot = errors.New("ActorTemplate has no golden snapshot") + +// resolveGoldenSnapshot resolves the storage location of the template's +// golden snapshot, validating that it can serve as a boot source or as the +// guest half of a combined restore. Returns errNoGoldenSnapshot when the +// template records no golden snapshot. +func resolveGoldenSnapshot(actorTemplate *ateapipb.ActorTemplate) (resources.SnapshotURI, error) { + golden := actorTemplate.GetStatus().GetGoldenSnapshotStatus().GetGoldenSnapshot() + if golden.GetSnapshotUri() == "" { + return resources.SnapshotURI{}, errNoGoldenSnapshot + } + if err := validateGoldenSnapshotScope(golden); err != nil { + return resources.SnapshotURI{}, err + } + uri, err := resources.ParseSnapshotURI(golden.GetSnapshotUri()) + if err != nil { + return resources.SnapshotURI{}, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", golden.GetSnapshotUri(), err) + } + return uri, nil +} + +// resolveGoldenForDataResume resolves the golden snapshot a Golden data +// resume rides, failing with FailedPrecondition when the template records +// none. +func resolveGoldenForDataResume(actorTemplate *ateapipb.ActorTemplate) (resources.SnapshotURI, error) { + uri, err := resolveGoldenSnapshot(actorTemplate) + if errors.Is(err, errNoGoldenSnapshot) { + return resources.SnapshotURI{}, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") + } + return uri, err +} + // ensureVolumesCreated provisions any initial actor volumes that are in // PENDING state, persisting the resulting volume state (even when creation // partially failed, so progress is not lost) and returning the stored copy. @@ -690,19 +768,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou req.Config = &ateletpb.RestoreRequest_LocalConfig{ LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotName: local.GetSnapshotName()}, } - // The wire scope describes the restore OPERATION: DATA_ON_GOLDEN when - // loadActorForResume resolved a golden URI per the template's onResume - // configuration, else what the pause captured. - switch { - case src.TemplateReplaced: - req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA - case !src.GoldenSnapshotURI.IsZero(): - req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN - req.GoldenSnapshotUri = src.GoldenSnapshotURI.String() - default: - req.Scope = actorSnapshotContentScopeToAtelet(actorTemplate.GetSnapshotsConfig().GetOnPause()) + req.Scope = src.WireScope + if src.WireScope == ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN { + req.GoldenSnapshotUri = src.GoldenForDataSnapshotURI.String() } - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope) + tele.WireSnapshotScope = ateattr.SnapshotScopeValue(src.WireScope) _, err = client.Restore(ctx, req) return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring workload", ateattr.OperationResume) @@ -714,18 +784,11 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "" { tele.SnapshotKind = ateattr.SnapshotKindLatest } - var scope ateletpb.SnapshotScope var goldenSnapshotURI string - switch { - case src.TemplateReplaced: - scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA - case !src.GoldenSnapshotURI.IsZero(): - scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN - goldenSnapshotURI = src.GoldenSnapshotURI.String() - default: - scope = actorSnapshotContentScopeToAtelet(src.Scope) + if src.WireScope == ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN { + goldenSnapshotURI = src.GoldenForDataSnapshotURI.String() } - tele.WireSnapshotScope = ateattr.SnapshotScopeValue(scope) + tele.WireSnapshotScope = ateattr.SnapshotScopeValue(src.WireScope) req := &ateletpb.RestoreRequest{ TargetAteomUid: assignment.GetWorkerPodUid(), Atespace: actor.GetMetadata().GetAtespace(), @@ -739,7 +802,7 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou SnapshotUri: src.SnapshotURI.String(), }, }, - Scope: scope, + Scope: src.WireScope, // Empty unless this is a Golden data resume. GoldenSnapshotUri: goldenSnapshotURI, ActorUid: actor.GetMetadata().Uid, diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index aa692ec4b1..d9468f401e 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" @@ -910,6 +911,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope ateapipb.SnapshotContentScope wantCode codes.Code wantGoldenURI string + wantWireScope ateletpb.SnapshotScope }{ { name: "resolves golden location for Data durable snapshot", @@ -919,6 +921,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, wantCode: codes.OK, wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, { name: "resolves golden location for paused actor with Data onPause", @@ -929,6 +932,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, wantCode: codes.OK, wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, { // A Full pause snapshot restores from its own content; the policy @@ -941,6 +945,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, wantCode: codes.OK, wantGoldenURI: "", + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, }, { name: "fails when golden snapshot is not Full", @@ -974,6 +979,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, wantCode: codes.OK, wantGoldenURI: "", + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, }, { name: "leaves golden location empty under ColdBoot fromData", @@ -983,6 +989,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, wantCode: codes.OK, wantGoldenURI: "", + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, }, } @@ -1035,11 +1042,11 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { if err != nil { return } - if got := src.GoldenSnapshotURI.String(); got != tt.wantGoldenURI { - t.Errorf("src.GoldenSnapshotURI = %q, want %q", got, tt.wantGoldenURI) + if got := src.GoldenForDataSnapshotURI.String(); got != tt.wantGoldenURI { + t.Errorf("src.GoldenForDataSnapshotURI = %q, want %q", got, tt.wantGoldenURI) } - if !tt.paused && src.Scope != tt.contentScope { - t.Errorf("src.Scope = %v, want %v", src.Scope, tt.contentScope) + if src.WireScope != tt.wantWireScope { + t.Errorf("src.WireScope = %v, want %v", src.WireScope, tt.wantWireScope) } }) } @@ -1085,8 +1092,8 @@ func TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden(t *testing.T) { // TestLoadActorForResume_TemplateReplaced covers the detection of a repointed // actor: the actor records the template UID its guest state was built on, and -// a mismatch with its current template marks the source TemplateReplaced, -// forcing the restore to data-only. +// a mismatch with its current template means the snapshot's guest state must +// not be restored, forcing the restore operation to data-only. func TestLoadActorForResume_TemplateReplaced(t *testing.T) { actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} @@ -1097,12 +1104,15 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { // "" leaves the field unset (an actor from before it was recorded). builtOnTemplateUID string noSnapshot bool - want bool + // wantWireScope is the resolved restore scope: DATA for a repointed + // actor's Full snapshot, FULL for a snapshot restored as-is, and + // unspecified (inert) for a cold boot with no snapshot or golden. + wantWireScope ateletpb.SnapshotScope }{ - {name: "snapshot taken under the current template", builtOnTemplateUID: "current", want: false}, - {name: "snapshot taken under a replaced template", builtOnTemplateUID: "some-other-uid", want: true}, - {name: "snapshot without a recorded template UID", builtOnTemplateUID: "", want: false}, - {name: "no durable snapshot", noSnapshot: true, want: false}, + {name: "snapshot taken under the current template", builtOnTemplateUID: "current", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, + {name: "snapshot taken under a replaced template", builtOnTemplateUID: "some-other-uid", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA}, + {name: "snapshot without a recorded template UID", builtOnTemplateUID: "", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, + {name: "no durable snapshot", noSnapshot: true, wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED}, } for _, tt := range tests { @@ -1142,8 +1152,121 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { if err != nil { t.Fatalf("loadActorForResume: %v", err) } - if src.TemplateReplaced != tt.want { - t.Errorf("src.TemplateReplaced = %v, want %v", src.TemplateReplaced, tt.want) + if src.WireScope != tt.wantWireScope { + t.Errorf("src.WireScope = %v, want %v", src.WireScope, tt.wantWireScope) + } + }) + } +} + +// TestLoadActorForResume_BootForcesDataScope covers the explicit boot +// request: the restore discards any captured guest state and carries the +// actor's durable data alone, so the wire scope is Data regardless of the +// snapshot's captured scope, the pause scope, or the onResume policy — and +// no golden snapshot is resolved (an unusable golden must not block a boot). +func TestLoadActorForResume_BootForcesDataScope(t *testing.T) { + goldenSnapshotURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") + actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + + tests := []struct { + name string + // paused seeds a local pause checkpoint; noSnapshot leaves the actor + // without a durable snapshot; otherwise a durable snapshot captured + // with contentScope is seeded. + paused bool + noSnapshot bool + contentScope ateapipb.SnapshotContentScope + // fromData seeds the template's onResume policy; goldenURI the + // template's recorded golden snapshot. + fromData ateapipb.ResumeSource + goldenURI string + // wantSnapshotURI reports whether the actor's own durable snapshot + // must be the boot source; a boot never resolves the golden as one. + wantSnapshotURI bool + }{ + { + name: "Full durable snapshot resumes data-only", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantSnapshotURI: true, + }, + { + name: "Data durable snapshot skips the golden ride under the Golden policy", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + goldenURI: goldenSnapshotURI, + wantSnapshotURI: true, + }, + { + name: "no snapshot skips the golden fallback and cold boots", + noSnapshot: true, + goldenURI: goldenSnapshotURI, + }, + { + name: "paused actor restores its checkpoint data-only", + paused: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + + durableSnapshotURI := someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1") + var seedOpts []func(*ateapipb.Actor) + if !tt.noSnapshot && !tt.paused { + seedOpts = append(seedOpts, func(a *ateapipb.Actor) { + a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ + SnapshotUri: durableSnapshotURI, + ContentScope: tt.contentScope, + } + }) + } + actorState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + if tt.paused { + actorState = ateapipb.ActorState_ACTOR_STATE_PAUSED + seedOpts = append(seedOpts, func(a *ateapipb.Actor) { + a.Status.LocalSnapshotInfo = &ateapipb.LocalSnapshotInfo{SnapshotName: "pause-1"} + }) + } + seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, seedOpts...) + + storetest.MustCreateAtespace(t, ctx, persistence, "ns") + tmpl := &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, + SnapshotsConfig: &ateapipb.SnapshotsConfig{ + OnResume: &ateapipb.OnResumeConfig{FromData: tt.fromData}, + }, + } + if tt.goldenURI != "" { + tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ + GoldenSnapshot: &ateapipb.ExternalSnapshot{ + SnapshotUri: tt.goldenURI, + ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + }, + }} + } + if _, err := persistence.CreateActorTemplate(ctx, tmpl); err != nil { + t.Fatalf("create template: %v", err) + } + + w := &ActorWorkflow{store: persistence} + _, _, src, err := w.loadActorForResume(ctx, actorRef, true) + if err != nil { + t.Fatalf("loadActorForResume: %v", err) + } + wantSnapshotURI := "" + if tt.wantSnapshotURI { + wantSnapshotURI = durableSnapshotURI + } + if got := src.SnapshotURI.String(); got != wantSnapshotURI { + t.Errorf("src.SnapshotURI = %q, want %q", got, wantSnapshotURI) + } + if !src.GoldenForDataSnapshotURI.IsZero() { + t.Errorf("src.GoldenForDataSnapshotURI = %q, want empty", src.GoldenForDataSnapshotURI) + } + if src.WireScope != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + t.Errorf("src.WireScope = %v, want %v", src.WireScope, ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA) } }) } @@ -1171,7 +1294,7 @@ func TestLoadActorForResume_RunningActorShortCircuits(t *testing.T) { if tmpl != nil { t.Errorf("expected nil template, got %v", tmpl) } - if !src.SnapshotURI.IsZero() || !src.GoldenSnapshotURI.IsZero() { + if !src.SnapshotURI.IsZero() || !src.GoldenForDataSnapshotURI.IsZero() { t.Errorf("expected empty snapshot source, got %+v", src) } } From 75b6b677b0b7fd1c56eced4d47be6cb1c440d870 Mon Sep 17 00:00:00 2001 From: zoezhao Date: Fri, 4 Sep 2026 14:57:41 -0700 Subject: [PATCH 2/2] Resume a repointed actor on its new template's golden snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A repointed actor's snapshot holds guest state captured under the replaced template, so it must never be restored as-is. Previously every repoint fell back to a data-only restore, discarding the new template's golden snapshot entirely. ateapi now resolves a repointed resume against the new template, based on the actor's last capture: - A Full (or legacy Unspecified) capture — durable snapshot or pause checkpoint — rides the new template's golden snapshot combined with the actor's durable data (DATA_ON_GOLDEN), and fails the resume with FailedPrecondition when that golden is not yet available instead of silently discarding the guest state. - A Data-scoped capture holds no guest state to lose, so it follows the onResume policy like any data resume: data-only cold boot by default, riding the golden under the Golden policy. - An actor with no snapshot at all restores the new template's golden as its own Full content, and cold boots from the spec when the template has none. atelet's Restore narrows the actor's half of a DATA_ON_GOLDEN combine to its durable data before staging: a FULL actor snapshot also lists guest files whose names would shadow the golden's in the combined set, which would resurrect the replaced template's guest state. An actor with no durable data now contributes nothing and boots the golden alone. The updatetemplate e2e suite covers the repoint lifecycle end to end: the resume after a repoint restores the durable data on the new template's golden, and a later snapshot taken under the new template leaves the repoint path behind. The counter demo gains a boot UUID that lives only in guest memory, so the test can tell a restored guest from a fresh boot. --- .../internal/controlapi/workflow_resume.go | 228 ++++-- .../controlapi/workflow_resume_test.go | 651 +++++++++++++++--- cmd/atelet/main.go | 19 + cmd/atelet/main_test.go | 44 ++ demos/counter/counter.go | 9 +- .../e2e/suites/demo/updatetemplate_test.go | 152 +++- 6 files changed, 922 insertions(+), 181 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 082e03903d..64e1498f3a 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -43,10 +43,12 @@ type resumeSnapshotSource struct { // the actor holds a local snapshot, which takes precedence at restore). SnapshotURI resources.SnapshotURI // GoldenForDataSnapshotURI is the storage location of the ActorTemplate's - // golden snapshot. Populated only when the template's onResume - // configuration selects the golden snapshot for a pending data-only - // restore: the restore then combines the golden snapshot with the - // actor's data. + // golden snapshot. Populated only when the pending restore must ride the + // golden: + // - the actor was repointed to a replacement template, so its snapshot's + // guest state was captured under the replaced one, or + // - the template's onResume configuration selects the golden for a + // data-only restore. GoldenForDataSnapshotURI resources.SnapshotURI // WireScope is the scope of the restore operation, not of any stored // snapshot: @@ -166,13 +168,14 @@ func validateGoldenSnapshotScope(snapshot *ateapipb.ExternalSnapshot) error { // 1. An explicit boot request restores data-only from whatever snapshot // exists (none: cold boot from the spec), never riding a golden. // 2. A repointed actor's snapshot holds guest state from the replaced -// template, so the restore carries the actor's durable data alone. +// template, so only its durable data survives; see +// resolveResumeScopeAfterUpdatedTemplate. // 3. A paused actor restores its local pause checkpoint at the template's -// onPause scope; a Data-scoped pause rides the golden when the onResume -// policy selects it. -// 4. An actor on its own template restores its snapshot at the captured -// scope; a Data-scoped capture rides the golden when the onResume -// policy selects it. +// onPause scope; a Data-scoped checkpoint rides the golden when the +// onResume policy selects it. +// 4. An actor with a durable snapshot restores it at the captured scope; +// a Data-scoped capture rides the golden when the onResume policy +// selects it. // 5. An actor with no snapshot restores the template's golden when one // exists, and cold boots from the spec otherwise. func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resources.ActorRef, boot bool) (_ *ateapipb.Actor, _ *ateapipb.ActorTemplate, _ resumeSnapshotSource, err error) { @@ -199,12 +202,11 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if err != nil { return nil, nil, src, err } - external := actor.GetStatus().GetExternalSnapshot() - if uri := external.GetSnapshotUri(); uri != "" { - if src.SnapshotURI, err = resources.ParseSnapshotURI(uri); err != nil { - return nil, nil, src, status.Errorf(codes.DataLoss, "Actor %s external snapshot: %v", actorRef, err) - } + lastDurableSnapshot, err := resolveActorLatestDurableSnapshot(actor) + if err != nil { + return nil, nil, src, err } + src.SnapshotURI = lastDurableSnapshot.uri // Rule 1: an explicit boot request discards any captured guest state and // carries the actor's durable data alone (none: cold boot from the spec). @@ -214,11 +216,11 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc } onResumeGolden := actorTemplate.GetSnapshotsConfig().GetOnResume().GetFromData() == ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN - // Rule 2: An actor whose template was updated will cold boot. - builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid() - if external.GetSnapshotUri() != "" && builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid() { - src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA - return actor, actorTemplate, src, nil + + // Rule 2: the actor was repointed, so its snapshot's guest state belongs + // to the replaced template. + if isActorTemplateReplaced(actorTemplate, actor) { + return resolveResumeScopeAfterUpdatedTemplate(actor, actorTemplate, lastDurableSnapshot) } // Rule 3: a paused actor restores its local pause checkpoint, which the @@ -226,9 +228,15 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc if actor.GetStatus().GetLocalSnapshotInfo() != nil { if actorTemplate.GetSnapshotsConfig().GetOnPause() == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { if onResumeGolden { - if src.GoldenForDataSnapshotURI, err = resolveGoldenForDataResume(actorTemplate); err != nil { + golden, err := resolveGoldenSnapshot(actorTemplate) + if errors.Is(err, errNoGoldenSnapshot) { + return nil, nil, src, status.Error(codes.FailedPrecondition, + "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") + } + if err != nil { return nil, nil, src, err } + src.GoldenForDataSnapshotURI = golden.uri src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN return actor, actorTemplate, src, nil } @@ -242,12 +250,18 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc // Rule 4: the actor restores its own durable snapshot at the captured // scope; the onResume policy only governs data-only restores, so a valid // Full (or legacy Unspecified) capture restores from its own content. - if external.GetSnapshotUri() != "" { - if external.GetContentScope() == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { + if lastDurableSnapshot.exists { + if lastDurableSnapshot.capturedScope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA { if onResumeGolden { - if src.GoldenForDataSnapshotURI, err = resolveGoldenForDataResume(actorTemplate); err != nil { + golden, err := resolveGoldenSnapshot(actorTemplate) + if errors.Is(err, errNoGoldenSnapshot) { + return nil, nil, src, status.Error(codes.FailedPrecondition, + "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") + } + if err != nil { return nil, nil, src, err } + src.GoldenForDataSnapshotURI = golden.uri src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN return actor, actorTemplate, src, nil } @@ -257,57 +271,152 @@ func (w *ActorWorkflow) loadActorForResume(ctx context.Context, actorRef resourc src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL return actor, actorTemplate, src, nil } - // Rule 5: an actor with no snapshot of its own restores the template's // golden snapshot as its own content when one exists, and cold boots // from the spec otherwise (the wire scope is inert then: RunRequest // carries no scope). - goldenURI, err := resolveGoldenSnapshot(actorTemplate) + golden, err := resolveGoldenSnapshot(actorTemplate) if errors.Is(err, errNoGoldenSnapshot) { return actor, actorTemplate, src, nil } if err != nil { return nil, nil, src, err } - src.SnapshotURI = goldenURI + src.SnapshotURI = golden.uri // A golden snapshot is always FULL scope. src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL return actor, actorTemplate, src, nil } +// resolveResumeScopeAfterUpdatedTemplate resolves the boot source for an +// actor repointed to a replacement template, based on its last snapshot: +// - No snapshot at all: restore the new template's golden snapshot (cold +// boot from the spec when there is none). +// - Data-scoped: ride the new template's golden when onResume.fromData +// selects it, restore data-only otherwise. +// - Full-scoped: the guest state belongs to the replaced template, so +// downgrade to data on the new template's golden. +func resolveResumeScopeAfterUpdatedTemplate(actor *ateapipb.Actor, actorTemplate *ateapipb.ActorTemplate, lastDurableSnapshot durableSnapshot) (*ateapipb.Actor, *ateapipb.ActorTemplate, resumeSnapshotSource, error) { + var src resumeSnapshotSource + + // The actor does not have any snapshot, so nothing captured under the + // replaced template carries over: restore the new template's golden + // snapshot as the actor's own Full content when one exists, and cold + // boot from the spec otherwise (the Full scope is inert then: + // RunRequest carries no scope). + if !lastDurableSnapshot.exists && actor.GetStatus().GetLocalSnapshotInfo() == nil { + golden, err := resolveGoldenSnapshot(actorTemplate) + switch { + case errors.Is(err, errNoGoldenSnapshot): + // Cold boot from the spec; the Full scope below is inert then. + case err != nil: + return nil, nil, src, err + default: + src.SnapshotURI = golden.uri + } + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL + return actor, actorTemplate, src, nil + } + + // The actor has a local or a durable snapshot; resolve the scope its last + // snapshot was captured with. A local snapshot only exists for a paused + // actor, and its scope is the template's onPause scope. + var lastSnapshotScope ateapipb.SnapshotContentScope + if actor.GetStatus().GetLocalSnapshotInfo() != nil { + lastSnapshotScope = actorTemplate.GetSnapshotsConfig().GetOnPause() + } else { + lastSnapshotScope = lastDurableSnapshot.capturedScope + } + + src.SnapshotURI = lastDurableSnapshot.uri + + // A Data-scoped capture carried no guest state to lose, so it follows the + // onResume policy like any data resume: data-only unless the policy + // selects the golden. + if lastSnapshotScope == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA && + actorTemplate.GetSnapshotsConfig().GetOnResume().GetFromData() != ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN { + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + return actor, actorTemplate, src, nil + } + + // A Full capture (its guest state belongs to the replaced template), or a + // Data capture whose onResume policy selects the golden: ride the new + // template's golden snapshot. + golden, err := resolveGoldenSnapshot(actorTemplate) + if errors.Is(err, errNoGoldenSnapshot) { + return nil, nil, src, status.Error(codes.FailedPrecondition, + "a repointed actor's resume requires the new ActorTemplate's golden snapshot, which is not available") + } + if err != nil { + return nil, nil, src, err + } + src.GoldenForDataSnapshotURI = golden.uri + src.WireScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN + return actor, actorTemplate, src, nil +} + +// isActorTemplateReplaced reports whether the actor was repointed: the actor +// records the template UID its guest state was built on, and a different UID +// on its current template means it was repointed since the capture. +func isActorTemplateReplaced(actorTemplate *ateapipb.ActorTemplate, actor *ateapipb.Actor) bool { + builtOnTemplateUID := actor.GetStatus().GetCurrentActorTemplateUid() + return builtOnTemplateUID != "" && builtOnTemplateUID != actorTemplate.GetMetadata().GetUid() +} + +// durableSnapshot is a resolved external snapshot record (the actor's own +// latest durable snapshot, or the template's golden), resolved once per +// resume. +type durableSnapshot struct { + // exists reports whether the record names a snapshot at all. + exists bool + // uri is the storage location of the snapshot's content. + uri resources.SnapshotURI + // capturedScope is the content scope the snapshot was taken with. + capturedScope ateapipb.SnapshotContentScope +} + +// resolveActorLatestDurableSnapshot resolves the actor's latest durable +// snapshot, when one exists, from the external snapshot the actor records. +func resolveActorLatestDurableSnapshot(actor *ateapipb.Actor) (durableSnapshot, error) { + var d durableSnapshot + external := actor.GetStatus().GetExternalSnapshot() + if external.GetSnapshotUri() == "" { + return d, nil + } + var err error + if d.uri, err = resources.ParseSnapshotURI(external.GetSnapshotUri()); err != nil { + return d, status.Errorf(codes.DataLoss, "Actor %s/%s external snapshot: %v", + actor.GetMetadata().GetAtespace(), actor.GetMetadata().GetName(), err) + } + d.exists = true + d.capturedScope = external.GetContentScope() + return d, nil +} + // errNoGoldenSnapshot reports that the ActorTemplate records no golden // snapshot. Callers decide what that means for their resume path: a gRPC // status naming the path that needed the golden, or a fallback source. var errNoGoldenSnapshot = errors.New("ActorTemplate has no golden snapshot") -// resolveGoldenSnapshot resolves the storage location of the template's -// golden snapshot, validating that it can serve as a boot source or as the -// guest half of a combined restore. Returns errNoGoldenSnapshot when the -// template records no golden snapshot. -func resolveGoldenSnapshot(actorTemplate *ateapipb.ActorTemplate) (resources.SnapshotURI, error) { +// resolveGoldenSnapshot resolves the template's golden snapshot, validating +// that it can serve as the guest half of a combined restore. Returns +// errNoGoldenSnapshot when the template records no golden snapshot. +func resolveGoldenSnapshot(actorTemplate *ateapipb.ActorTemplate) (durableSnapshot, error) { golden := actorTemplate.GetStatus().GetGoldenSnapshotStatus().GetGoldenSnapshot() if golden.GetSnapshotUri() == "" { - return resources.SnapshotURI{}, errNoGoldenSnapshot + return durableSnapshot{}, errNoGoldenSnapshot } if err := validateGoldenSnapshotScope(golden); err != nil { - return resources.SnapshotURI{}, err + return durableSnapshot{}, err } - uri, err := resources.ParseSnapshotURI(golden.GetSnapshotUri()) - if err != nil { - return resources.SnapshotURI{}, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", golden.GetSnapshotUri(), err) + var d durableSnapshot + var err error + if d.uri, err = resources.ParseSnapshotURI(golden.GetSnapshotUri()); err != nil { + return durableSnapshot{}, status.Errorf(codes.DataLoss, "golden external snapshot %q: %v", golden.GetSnapshotUri(), err) } - return uri, nil -} - -// resolveGoldenForDataResume resolves the golden snapshot a Golden data -// resume rides, failing with FailedPrecondition when the template records -// none. -func resolveGoldenForDataResume(actorTemplate *ateapipb.ActorTemplate) (resources.SnapshotURI, error) { - uri, err := resolveGoldenSnapshot(actorTemplate) - if errors.Is(err, errNoGoldenSnapshot) { - return resources.SnapshotURI{}, status.Error(codes.FailedPrecondition, "a Golden data resume requires the ActorTemplate golden snapshot, which is not available") - } - return uri, err + d.exists = true + d.capturedScope = golden.GetContentScope() + return d, nil } // ensureVolumesCreated provisions any initial actor volumes that are in @@ -784,10 +893,6 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou if actor.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "" { tele.SnapshotKind = ateattr.SnapshotKindLatest } - var goldenSnapshotURI string - if src.WireScope == ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN { - goldenSnapshotURI = src.GoldenForDataSnapshotURI.String() - } tele.WireSnapshotScope = ateattr.SnapshotScopeValue(src.WireScope) req := &ateletpb.RestoreRequest{ TargetAteomUid: assignment.GetWorkerPodUid(), @@ -802,13 +907,16 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou SnapshotUri: src.SnapshotURI.String(), }, }, - Scope: src.WireScope, - // Empty unless this is a Golden data resume. - GoldenSnapshotUri: goldenSnapshotURI, - ActorUid: actor.GetMetadata().Uid, - EgressGateway: egressGateway, - CpuMilli: cpuMilli, - MemoryBytes: memBytes, + Scope: src.WireScope, + ActorUid: actor.GetMetadata().Uid, + EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, + } + // Set only when the restore rides the golden snapshot (a Golden data + // resume, or a repointed actor). + if src.WireScope == ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN { + req.GoldenSnapshotUri = src.GoldenForDataSnapshotURI.String() } _, err = client.Restore(ctx, req) return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring durable snapshot", ateattr.OperationResume) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index d9468f401e..408d8c5fb0 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "testing" "time" @@ -888,9 +887,12 @@ func TestValidateAssignedWorker_WorkerOwnership(t *testing.T) { } // TestLoadActorForResume_OnGoldenDataResume verifies the golden-location -// plumbing: when the template's onResume.fromData is Golden, a pending -// data-only restore (a Data durable snapshot, or a paused actor whose -// onPause is Data) additionally resolves the template's golden snapshot +// plumbing and the resolved wire scope: when the template's onResume.fromData +// is Golden, a pending data-only restore (a Data durable snapshot, or a +// paused actor whose onPause is Data) additionally resolves the template's +// golden snapshot and rides it as DATA_ON_GOLDEN; otherwise the captured +// scope of the snapshot the restore will use rides out, the local pause +// checkpoint taking precedence over the durable snapshot. func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { goldenSnapshotURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} @@ -899,11 +901,18 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { name string fromData ateapipb.ResumeSource // paused seeds the actor with LocalSnapshotInfo (a pause checkpoint) - // instead of a durable snapshot; onPause is the template's pause - // scope, contentScope the durable snapshot's recorded content. + // instead of a durable snapshot (in addition to one when alsoDurable + // is set); onPause is the template's pause scope, contentScope the + // durable snapshot's recorded content. paused bool + alsoDurable bool onPause ateapipb.SnapshotContentScope contentScope ateapipb.SnapshotContentScope + // builtOnTemplateUID seeds the template UID the actor's guest state + // was built on; a non-empty value differing from the created + // template's own UID marks the actor repointed. Empty leaves the + // field unset. + builtOnTemplateUID string // goldenURI and goldenScope are the template's recorded golden // external snapshot; an empty URI means the template has none. A zero // scope is treated as Full, the scope a golden snapshot must hold. @@ -934,6 +943,21 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { wantGoldenURI: goldenSnapshotURI, wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, + { + // A repointed actor's data resume resolves the replacement + // template's golden through the repoint path, which takes + // precedence over the onResume policy but lands on the same + // golden location. + name: "resolves golden location for repointed Data durable snapshot", + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + builtOnTemplateUID: "replaced-uid", + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, { // A Full pause snapshot restores from its own content; the policy // only governs data-only restores. @@ -947,6 +971,32 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { wantGoldenURI: "", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, }, + { + // The restore uses the local pause checkpoint, so its Full onPause + // scope decides — the stale durable Data snapshot neither pulls in + // the golden nor narrows the wire scope. + name: "local pause scope wins over a stale durable Data snapshot", + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + paused: true, + alsoDurable: true, + onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantGoldenURI: "", + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + // Legacy snapshots recorded before content_scope existed read as + // Full on the wire. + name: "unspecified captured scope reads as Full", + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, + wantCode: codes.OK, + wantGoldenURI: "", + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, { name: "fails when golden snapshot is not Full", fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, @@ -1003,12 +1053,14 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { seedOpts = append(seedOpts, func(a *ateapipb.Actor) { a.Status.LocalSnapshotInfo = &ateapipb.LocalSnapshotInfo{SnapshotName: "pause-1"} }) - } else { + } + if !tt.paused || tt.alsoDurable { seedOpts = append(seedOpts, func(a *ateapipb.Actor) { a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), ContentScope: tt.contentScope, } + a.Status.CurrentActorTemplateUid = tt.builtOnTemplateUID }) } actorState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED @@ -1043,7 +1095,7 @@ func TestLoadActorForResume_OnGoldenDataResume(t *testing.T) { return } if got := src.GoldenForDataSnapshotURI.String(); got != tt.wantGoldenURI { - t.Errorf("src.GoldenForDataSnapshotURI = %q, want %q", got, tt.wantGoldenURI) + t.Errorf("src.GoldenSnapshotURI = %q, want %q", got, tt.wantGoldenURI) } if src.WireScope != tt.wantWireScope { t.Errorf("src.WireScope = %v, want %v", src.WireScope, tt.wantWireScope) @@ -1085,34 +1137,270 @@ func TestLoadActorForResume_GoldenFallbackRejectsNonFullGolden(t *testing.T) { if got := status.Code(err); got != codes.FailedPrecondition { t.Fatalf("status.Code(err) = %v, want FailedPrecondition (err: %v)", got, err) } - if !strings.Contains(err.Error(), "regenerate the golden snapshot") { - t.Errorf("error %q does not tell the operator to regenerate the golden snapshot", err) - } } -// TestLoadActorForResume_TemplateReplaced covers the detection of a repointed -// actor: the actor records the template UID its guest state was built on, and -// a mismatch with its current template means the snapshot's guest state must -// not be restored, forcing the restore operation to data-only. -func TestLoadActorForResume_TemplateReplaced(t *testing.T) { +// TestLoadActorForResume_Repointed covers the resume of a repointed actor: +// when the guest state the restore will boot from was captured under a +// template other than the actor's current one, it belongs to the replaced +// template and the restore must ride the replacement template's golden +// snapshot — for Full and Data snapshots alike — and fail when that golden +// is not usable. A new template whose onResume policy selects ColdBoot, or +// an explicit boot request, instead resumes data-only, with no golden +// required. The actor's template stamp (Status.CurrentActorTemplateUid) is +// the detection signal; actors without one (never committed RUNNING) restore +// themselves. +func TestLoadActorForResume_Repointed(t *testing.T) { + goldenSnapshotURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + durableSnapshotURI := someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1") tests := []struct { name string - // builtOnTemplateUID seeds the template UID the actor's guest state + // builtOnTemplateUID seeds the actor's template stamp + // (Status.CurrentActorTemplateUid), the template UID its guest state // was built on; "current" stands for the created template's own UID, - // "" leaves the field unset (an actor from before it was recorded). + // "" leaves the stamp unset (an actor that never committed RUNNING: + // created from scratch or from a snapshot clone). builtOnTemplateUID string + contentScope ateapipb.SnapshotContentScope noSnapshot bool - // wantWireScope is the resolved restore scope: DATA for a repointed - // actor's Full snapshot, FULL for a snapshot restored as-is, and - // unspecified (inert) for a cold boot with no snapshot or golden. - wantWireScope ateletpb.SnapshotScope + // paused seeds the actor PAUSED with a local pause snapshot, the + // boot source that takes precedence over any durable snapshot. + paused bool + // goldenURI and goldenScope are the template's recorded golden + // external snapshot; an empty URI means the template has none. + goldenURI string + goldenScope ateapipb.SnapshotContentScope + // fromData seeds the template's onResume policy; UNSPECIFIED leaves + // the config unset. + fromData ateapipb.ResumeSource + // boot is the caller's explicit cold-boot request. + boot bool + wantCode codes.Code + // wantSnapshotURI is the boot snapshot the resume must resolve: the + // actor's own durable snapshot, the golden restored as the actor's + // own content, or empty for a cold boot from the spec. + wantSnapshotURI string + wantGoldenURI string + wantWireScope ateletpb.SnapshotScope }{ - {name: "snapshot taken under the current template", builtOnTemplateUID: "current", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, - {name: "snapshot taken under a replaced template", builtOnTemplateUID: "some-other-uid", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA}, - {name: "snapshot without a recorded template UID", builtOnTemplateUID: "", wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL}, - {name: "no durable snapshot", noSnapshot: true, wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED}, + { + // The headline fix: a repointed Full snapshot no longer restores + // (or cold-boots) its own guest state — it rides the new golden. + name: "repointed Full snapshot rides the new template's golden", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + // The onResume policy defaults to ColdBoot here; the repoint + // path resolves the golden anyway. + name: "repointed Data snapshot rides the golden without the Golden policy", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + // An explicit Golden policy resolves the golden the same way the + // unset default does. + name: "repointed Full snapshot rides the golden under an explicit Golden policy", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + // A ColdBoot policy resumes data-only: the snapshot's guest state + // is discarded, and no golden is required (none is seeded here). + name: "repointed Full snapshot resumes data-only under a ColdBoot policy", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "repointed Data snapshot resumes data-only under a ColdBoot policy", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "fails when the new template has no golden snapshot", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.FailedPrecondition, + }, + { + // The escape hatch for the case right above: an explicit boot + // request resumes data-only, so no golden is required. + name: "explicit boot resumes a repointed actor data-only without a golden", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + boot: true, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "explicit boot skips the golden even when one is available", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + boot: true, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "fails when the golden snapshot uri is malformed", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: "golden-1", + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.DataLoss, + }, + { + name: "fails when the golden snapshot is not Full", + builtOnTemplateUID: "some-other-uid", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + wantCode: codes.FailedPrecondition, + }, + { + name: "snapshot taken under the current template restores itself", + builtOnTemplateUID: "current", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + name: "snapshot without a recorded template UID restores itself", + builtOnTemplateUID: "", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + // A cold boot goes out as a RunRequest, which carries no scope; + // the resolved Full is inert. + name: "no durable snapshot resolves no golden", + noSnapshot: true, + wantCode: codes.OK, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + // The repointed no-snapshot case: nothing captured under the + // replaced template carries over, so the new template's golden + // boots as the actor's own Full content. + name: "repointed actor with no snapshot boots from the new template's golden", + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + name: "repointed actor with no snapshot cold boots when the template has no golden", + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + wantCode: codes.OK, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + name: "repointed actor with no snapshot fails when the golden uri is malformed", + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + goldenURI: "golden-1", + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.DataLoss, + }, + { + name: "repointed actor with no snapshot rejects a non-Full golden", + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + wantCode: codes.FailedPrecondition, + }, + { + // The local pause snapshot was captured under the current + // template (a repoint requires SUSPENDED, and suspending clears + // the local snapshot). It restores itself — no golden required, + // none seeded here. + name: "local pause snapshot under the current template restores itself", + paused: true, + builtOnTemplateUID: "current", + contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + // The one way a paused actor's template can change: deleted and + // recreated under the same name. The local snapshot's guest state + // belongs to the deleted template, so the restore rides the golden. + name: "local pause snapshot under a recreated template rides the golden", + paused: true, + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + name: "fails when the recreated template has no golden snapshot", + paused: true, + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + wantCode: codes.FailedPrecondition, + }, + { + name: "local pause snapshot under a recreated ColdBoot template resumes data-only", + paused: true, + builtOnTemplateUID: "some-other-uid", + noSnapshot: true, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT, + wantCode: codes.OK, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + // Actors paused before the template stamp existed cannot have + // been repointed while paused; they restore themselves. + name: "local pause snapshot without a template stamp restores itself", + paused: true, + builtOnTemplateUID: "", + noSnapshot: true, + wantCode: codes.OK, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, } for _, tt := range tests { @@ -1121,9 +1409,22 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { persistence := newTestPersistence(t) storetest.MustCreateAtespace(t, ctx, persistence, "ns") - tmpl, err := persistence.CreateActorTemplate(ctx, &ateapipb.ActorTemplate{ + toCreate := &ateapipb.ActorTemplate{ Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, - }) + } + if tt.fromData != ateapipb.ResumeSource_RESUME_SOURCE_UNSPECIFIED { + toCreate.SnapshotsConfig = &ateapipb.SnapshotsConfig{ + OnResume: &ateapipb.OnResumeConfig{FromData: tt.fromData}, + } + } + if tt.goldenURI != "" { + toCreate.Status = &ateapipb.ActorTemplateStatus{ + GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ + GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope}, + }, + } + } + tmpl, err := persistence.CreateActorTemplate(ctx, toCreate) if err != nil { t.Fatalf("create template: %v", err) } @@ -1133,24 +1434,44 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { var seedOpts []func(*ateapipb.Actor) if !tt.noSnapshot { + seedOpts = append(seedOpts, func(a *ateapipb.Actor) { + a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ + SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), + ContentScope: tt.contentScope, + } + }) + } + actorState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + if tt.paused { + actorState = ateapipb.ActorState_ACTOR_STATE_PAUSED + seedOpts = append(seedOpts, func(a *ateapipb.Actor) { + a.Status.LocalSnapshotInfo = &ateapipb.LocalSnapshotInfo{SnapshotName: "pause-1"} + }) + } + if tt.builtOnTemplateUID != "" { uid := tt.builtOnTemplateUID if uid == "current" { uid = tmpl.GetMetadata().GetUid() } seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1"), - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - } a.Status.CurrentActorTemplateUid = uid }) } - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", ateapipb.ActorState_ACTOR_STATE_SUSPENDED, seedOpts...) + seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, seedOpts...) w := &ActorWorkflow{store: persistence} - _, _, src, err := w.loadActorForResume(ctx, actorRef, false) + _, _, src, err := w.loadActorForResume(ctx, actorRef, tt.boot) + if got := status.Code(err); got != tt.wantCode { + t.Fatalf("status.Code(err) = %v, want %v (err: %v)", got, tt.wantCode, err) + } if err != nil { - t.Fatalf("loadActorForResume: %v", err) + return + } + if got := src.SnapshotURI.String(); got != tt.wantSnapshotURI { + t.Errorf("src.SnapshotURI = %q, want %q", got, tt.wantSnapshotURI) + } + if got := src.GoldenForDataSnapshotURI.String(); got != tt.wantGoldenURI { + t.Errorf("src.GoldenSnapshotURI = %q, want %q", got, tt.wantGoldenURI) } if src.WireScope != tt.wantWireScope { t.Errorf("src.WireScope = %v, want %v", src.WireScope, tt.wantWireScope) @@ -1159,114 +1480,230 @@ func TestLoadActorForResume_TemplateReplaced(t *testing.T) { } } -// TestLoadActorForResume_BootForcesDataScope covers the explicit boot -// request: the restore discards any captured guest state and carries the -// actor's durable data alone, so the wire scope is Data regardless of the -// snapshot's captured scope, the pause scope, or the onResume policy — and -// no golden snapshot is resolved (an unusable golden must not block a boot). -func TestLoadActorForResume_BootForcesDataScope(t *testing.T) { +// TestResolveResumeScopeAfterUpdatedTemplate covers the repointed-actor scope +// decision in isolation, with the durable snapshot record handed in directly: +// an actor with no snapshot at all boots the new template's golden as its own +// content; a Data-scoped capture follows the onResume policy (data-only by +// default, riding the golden when the policy selects it); any other capture +// holds the replaced template's guest state and must ride the golden. A +// paused actor's capture scope is the template's onPause policy, not the +// stale durable record's. +func TestResolveResumeScopeAfterUpdatedTemplate(t *testing.T) { goldenSnapshotURI := someActorSnapshotURI(t, "gs://bucket/golden-root", "ate-golden", "golden-1") - actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} + durableSnapshotURI := someActorSnapshotURI(t, testStorageLocation, "team-a", "snap-1") tests := []struct { name string - // paused seeds a local pause checkpoint; noSnapshot leaves the actor - // without a durable snapshot; otherwise a durable snapshot captured - // with contentScope is seeded. - paused bool - noSnapshot bool - contentScope ateapipb.SnapshotContentScope - // fromData seeds the template's onResume policy; goldenURI the - // template's recorded golden snapshot. - fromData ateapipb.ResumeSource - goldenURI string - // wantSnapshotURI reports whether the actor's own durable snapshot - // must be the boot source; a boot never resolves the golden as one. - wantSnapshotURI bool + // hasDurable hands in a durable snapshot captured with durableScope; + // without it the zero durableSnapshot stands for "none recorded". + hasDurable bool + durableScope ateapipb.SnapshotContentScope + // paused gives the actor a local pause snapshot; onPause is the + // template's pause scope, which then stands in as the capture scope. + paused bool + onPause ateapipb.SnapshotContentScope + // fromData seeds the template's onResume policy; UNSPECIFIED leaves + // it unset (ColdBoot semantics). + fromData ateapipb.ResumeSource + // goldenURI and goldenScope are the template's recorded golden + // external snapshot; an empty URI means the template has none. + goldenURI string + goldenScope ateapipb.SnapshotContentScope + wantCode codes.Code + // wantSnapshotURI is the resolved boot snapshot: the actor's own + // durable snapshot, the golden restored as the actor's own content, + // or empty for a cold boot from the spec. + wantSnapshotURI string + wantGoldenURI string + wantWireScope ateletpb.SnapshotScope }{ { - name: "Full durable snapshot resumes data-only", - contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, - wantSnapshotURI: true, + // Nothing captured under the replaced template carries over, so + // the new template's golden boots as the actor's own Full content. + name: "no snapshot boots the golden as the actor's own content", + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + // No snapshot and no golden: cold boot from the spec. The Full + // scope is inert — a cold boot goes out as a RunRequest, which + // carries no scope. + name: "no snapshot and no golden cold boots from the spec", + wantCode: codes.OK, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + { + name: "no snapshot fails when the golden uri is malformed", + goldenURI: "golden-1", + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.DataLoss, + }, + { + // A non-Full golden holds no guest state to boot from. + name: "no snapshot rejects a non-Full golden", + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + wantCode: codes.FailedPrecondition, }, { - name: "Data durable snapshot skips the golden ride under the Golden policy", - contentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + // A Data capture follows the onResume policy like any data + // resume; the unset default is ColdBoot, so no golden is needed + // (none is seeded here). + name: "Data snapshot resumes data-only under the default ColdBoot policy", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "Data snapshot rides the golden under the Golden policy", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, goldenURI: goldenSnapshotURI, - wantSnapshotURI: true, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, { - name: "no snapshot skips the golden fallback and cold boots", - noSnapshot: true, - goldenURI: goldenSnapshotURI, + // A Full capture holds the replaced template's guest state, so + // only its durable data survives: the restore must ride the new + // template's golden even without the Golden policy. + name: "Full snapshot rides the golden regardless of policy", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + // Legacy snapshots recorded before content_scope existed read as + // Full, so they ride the golden the same way. + name: "legacy Unspecified capture scope rides the golden like Full", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, + }, + { + name: "Full snapshot fails when the template has no golden", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.FailedPrecondition, + }, + { + name: "Full snapshot fails when the golden uri is malformed", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: "golden-1", + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.DataLoss, + }, + { + name: "Full snapshot rejects a non-Full golden", + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + wantCode: codes.FailedPrecondition, + }, + { + // The restore will use the local pause checkpoint, so the + // template's Data onPause decides — not the stale Full durable + // record — and the resume stays data-only with no golden needed. + name: "paused actor reads its scope from onPause, not the stale durable record", + paused: true, + onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + hasDurable: true, + durableScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantSnapshotURI: durableSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + }, + { + name: "paused Data onPause rides the golden under the Golden policy", + paused: true, + onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, + fromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, { - name: "paused actor restores its checkpoint data-only", - paused: true, + // A Full pause checkpoint carries the replaced template's guest + // state like any Full capture: it must ride the golden. + name: "paused Full onPause rides the golden", + paused: true, + onPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + goldenURI: goldenSnapshotURI, + goldenScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + wantCode: codes.OK, + wantGoldenURI: goldenSnapshotURI, + wantWireScope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - persistence := newTestPersistence(t) - - durableSnapshotURI := someActorSnapshotURI(t, testStorageLocation, actorRef.Atespace, "snap-1") - var seedOpts []func(*ateapipb.Actor) - if !tt.noSnapshot && !tt.paused { - seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.ExternalSnapshot = &ateapipb.ExternalSnapshot{ - SnapshotUri: durableSnapshotURI, - ContentScope: tt.contentScope, - } - }) - } - actorState := ateapipb.ActorState_ACTOR_STATE_SUSPENDED + actor := &ateapipb.Actor{Status: &ateapipb.ActorStatus{}} if tt.paused { - actorState = ateapipb.ActorState_ACTOR_STATE_PAUSED - seedOpts = append(seedOpts, func(a *ateapipb.Actor) { - a.Status.LocalSnapshotInfo = &ateapipb.LocalSnapshotInfo{SnapshotName: "pause-1"} - }) + actor.Status.LocalSnapshotInfo = &ateapipb.LocalSnapshotInfo{SnapshotName: "pause-1"} } - seedWorkflowActor(t, ctx, persistence, actorRef, "ns", "tmpl1", actorState, seedOpts...) - storetest.MustCreateAtespace(t, ctx, persistence, "ns") tmpl := &ateapipb.ActorTemplate{ - Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1"}, + Metadata: &ateapipb.ResourceMetadata{Atespace: "ns", Name: "tmpl1", Uid: "new-template-uid"}, SnapshotsConfig: &ateapipb.SnapshotsConfig{ + OnPause: tt.onPause, OnResume: &ateapipb.OnResumeConfig{FromData: tt.fromData}, }, } if tt.goldenURI != "" { - tmpl.Status = &ateapipb.ActorTemplateStatus{GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ - GoldenSnapshot: &ateapipb.ExternalSnapshot{ - SnapshotUri: tt.goldenURI, - ContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, + tmpl.Status = &ateapipb.ActorTemplateStatus{ + GoldenSnapshotStatus: &ateapipb.GoldenSnapshotStatus{ + GoldenSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: tt.goldenURI, ContentScope: tt.goldenScope}, }, - }} + } } - if _, err := persistence.CreateActorTemplate(ctx, tmpl); err != nil { - t.Fatalf("create template: %v", err) + + var durable durableSnapshot + if tt.hasDurable { + uri, err := resources.ParseSnapshotURI(durableSnapshotURI) + if err != nil { + t.Fatalf("ParseSnapshotURI(%q): %v", durableSnapshotURI, err) + } + durable = durableSnapshot{exists: true, uri: uri, capturedScope: tt.durableScope} } - w := &ActorWorkflow{store: persistence} - _, _, src, err := w.loadActorForResume(ctx, actorRef, true) - if err != nil { - t.Fatalf("loadActorForResume: %v", err) + _, _, src, err := resolveResumeScopeAfterUpdatedTemplate(actor, tmpl, durable) + if got := status.Code(err); got != tt.wantCode { + t.Fatalf("status.Code(err) = %v, want %v (err: %v)", got, tt.wantCode, err) } - wantSnapshotURI := "" - if tt.wantSnapshotURI { - wantSnapshotURI = durableSnapshotURI + if err != nil { + return } - if got := src.SnapshotURI.String(); got != wantSnapshotURI { - t.Errorf("src.SnapshotURI = %q, want %q", got, wantSnapshotURI) + if got := src.SnapshotURI.String(); got != tt.wantSnapshotURI { + t.Errorf("src.SnapshotURI = %q, want %q", got, tt.wantSnapshotURI) } - if !src.GoldenForDataSnapshotURI.IsZero() { - t.Errorf("src.GoldenForDataSnapshotURI = %q, want empty", src.GoldenForDataSnapshotURI) + if got := src.GoldenForDataSnapshotURI.String(); got != tt.wantGoldenURI { + t.Errorf("src.GoldenForDataSnapshotURI = %q, want %q", got, tt.wantGoldenURI) } - if src.WireScope != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { - t.Errorf("src.WireScope = %v, want %v", src.WireScope, ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA) + if src.WireScope != tt.wantWireScope { + t.Errorf("src.WireScope = %v, want %v", src.WireScope, tt.wantWireScope) } }) } diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 0314faa82e..fc8e5d25b7 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -942,6 +942,19 @@ func narrowFullCaptureToData(rec *sandboxAssetsRecord) error { } } +// narrowRestoreSetToData rewrites rec so it contributes only durable data to +// a DATA_ON_GOLDEN restore set. Unlike narrowFullCaptureToData, an absent +// durable-dir tar is not an error: an actor that holds no durable data rides +// the golden snapshot alone. +func narrowRestoreSetToData(rec *sandboxAssetsRecord) { + if slices.Contains(rec.SnapshotFiles, ateompath.DurableDirTarFile) { + rec.SnapshotFiles = []string{ateompath.DurableDirTarFile} + } else { + rec.SnapshotFiles = nil + } + rec.Scope = ateattr.SnapshotScopeData +} + func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (resp *ateletpb.RestoreResponse, err error) { if err := validateRestoreRequest(req); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -1078,6 +1091,12 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) if goldenRec.SandboxClass != sandboxRec.SandboxClass { return nil, status.Errorf(codes.FailedPrecondition, "golden snapshot sandbox class %q does not match actor snapshot sandbox class %q", goldenRec.SandboxClass, sandboxRec.SandboxClass) } + // The actor's half of a combined restore contributes only its durable + // data, but its snapshot may be a FULL capture (a repointed actor + // restores its commit snapshot on the new template's golden): its + // guest files would shadow the golden's in the combined set below, so + // narrow the actor's files to the durable data first. + narrowRestoreSetToData(sandboxRec) } dManifest = time.Since(tManifest) manifestDone = true diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 0e4ebb3cbf..616784b742 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1114,6 +1114,50 @@ func TestGoldenOnlyFiles(t *testing.T) { } } +// TestNarrowRestoreSetToData verifies the actor half of a DataOnGolden +// restore is reduced to its durable data: a FULL capture (a repointed actor's +// commit snapshot) lists guest files that would otherwise shadow the golden's +// in the combined set, and an actor with no durable data contributes nothing. +func TestNarrowRestoreSetToData(t *testing.T) { + tests := []struct { + name string + files []string + scope string + wantFiles []string + }{ + { + name: "full capture keeps only the durable tar", + files: []string{"config.json", "state.json", "memory-ranges", ateompath.DurableDirTarFile}, + scope: ateattr.SnapshotScopeFull, + wantFiles: []string{ateompath.DurableDirTarFile}, + }, + { + name: "data capture is unchanged", + files: []string{ateompath.DurableDirTarFile}, + scope: ateattr.SnapshotScopeData, + wantFiles: []string{ateompath.DurableDirTarFile}, + }, + { + name: "capture without durable data contributes nothing", + files: []string{"config.json", "state.json", "memory-ranges"}, + scope: ateattr.SnapshotScopeFull, + wantFiles: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rec := &sandboxAssetsRecord{SnapshotFiles: tc.files, Scope: tc.scope} + narrowRestoreSetToData(rec) + if diff := cmp.Diff(tc.wantFiles, rec.SnapshotFiles); diff != "" { + t.Errorf("SnapshotFiles diff (-want +got):\n%s", diff) + } + if rec.Scope != ateattr.SnapshotScopeData { + t.Errorf("Scope = %q, want %q", rec.Scope, ateattr.SnapshotScopeData) + } + }) + } +} + func TestWrapFileSystemErrAttachesTerminalReason(t *testing.T) { tests := []struct { name string diff --git a/demos/counter/counter.go b/demos/counter/counter.go index a3927a6dc8..7abb6200c8 100644 --- a/demos/counter/counter.go +++ b/demos/counter/counter.go @@ -43,6 +43,12 @@ var ( ready atomic.Bool fileMutex sync.Mutex sigtermSleepDurationSecs atomic.Int64 + + // bootUUID lives only in guest memory: generated before the server + // starts and never written to disk, so a response can only carry it if + // the serving process's memory descends from the boot that generated it. + // A checkpoint restore preserves it; a cold boot regenerates it. + bootUUID string ) func incrementFileCounter(filePath string) int { @@ -64,6 +70,7 @@ func incrementFileCounter(filePath string) int { } func main() { + bootUUID = rand.Text() sigtermSleepDurationSecs.Store(15) fileCounterDirectory := pflag.String("file-counter-directory", "/home/counter", "Directory for file counter") secondFileCounterDirectory := pflag.String("second-file-counter-directory", "", "Directory for a second file counter; empty disables it. Used to exercise an Actor with more than one durable volume") @@ -112,7 +119,7 @@ func main() { secondFileCounterStr = fmt.Sprintf(" | preserved second file counter: %d", secondFileCounter) } - response := fmt.Sprintf("hello from: %s | preserved memory count: %d | preserved file counter: %d%s%s\n", currentIP, memoryCounter, fileCounter, secondFileCounterStr, fileContentStr) + response := fmt.Sprintf("hello from: %s | preserved memory count: %d | preserved file counter: %d%s%s | boot uuid: %s\n", currentIP, memoryCounter, fileCounter, secondFileCounterStr, fileContentStr, bootUUID) slog.InfoContext(ctx, "Handled request", slog.String("response", response)) w.WriteHeader(http.StatusOK) diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 9729117a6b..8a44ce23c6 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -16,6 +16,7 @@ package demo import ( "context" + "regexp" "strings" "testing" "time" @@ -29,9 +30,15 @@ import ( // different ActorTemplate: the actor runs and writes to its durable-dir data // volume under template A, suspends, is repointed at template B via // UpdateActor, and resumes. The resume must detect the template change (the -// recorded current_actor_template_uid no longer matches) and restore -// data-only: the durable dir survives while the guest cold-boots from -// template B. +// snapshot's recorded template UID no longer matches) and restore the actor's +// durable data on template B's golden snapshot: the durable dir survives +// while the guest state comes from B's golden. The templates set +// onResume.fromData: Golden so that the Data-scope case rides the golden +// rather than cold-booting under the default policy. A final suspend/resume +// round trip under B then checks the repoint path is left behind: once a +// snapshot taken under B exists, the next resume restores that snapshot +// directly (including guest memory at Full scope) instead of taking the +// repoint path again. func TestUpdateTemplateLifecycle(t *testing.T) { tests := []struct { name string @@ -64,6 +71,9 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Fatalf("CheckEnv failed: %v", err) } + // + // Step 1: Create the two ActorTemplates the actor will move between. + // // Template A: the plain counter. Template B: the same workload, but its // command additionally validates that the file template A's sprint left // in the durable dir is readable — the response's "file content" both @@ -76,7 +86,7 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc }) // - // Create an Actor from template A; a fresh actor sits SUSPENDED. + // Step 2: Create an Actor from template A; a fresh actor sits SUSPENDED. // actorID := "update-" + nsObj.Name refA := &ateapipb.ObjectRef{Atespace: demoAtespace, Name: nameA} @@ -99,8 +109,11 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Fatalf("created Actor state = %v, want SUSPENDED", got) } - // Run under template A and write to the data volume (every call bumps the - // file counter the workload keeps in the durable dir). + // + // Step 3: Run under template A and write to the data volume (every call + // bumps both the in-memory counter and the file counter the workload + // keeps in the durable dir). + // t.Logf("Resuming Actor %q under template A...", actorID) if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -111,17 +124,23 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc // startup, so it gets a longer budget than the steady-state waits. waitForActorStateWithTimeout(ctx, t, clients, actorID, ateapipb.ActorState_ACTOR_STATE_RUNNING, 120*time.Second) + // A fresh actor's first resume rides its template's golden snapshot, so + // the boot uuid observed under A is A's golden's — the identity of the + // whole A-lineage guest state, including the snapshot the suspend below + // captures from it. + var goldenABootUUID string for i := 1; i <= 2; i++ { resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorID}) if err != nil { t.Fatalf("failed to call actor (call %d): %v", i, err) } validateCounterResponse(t, resp, "under template A", i, i) + goldenABootUUID = parseBootUUID(t, resp) } // - // Suspend; the record of the template the sprint booted with (stamped by - // the resume above) must survive the suspend. + // Step 4: Suspend; the record of the template the sprint booted with + // (stamped by the resume above) must survive the suspend. // t.Logf("Suspending Actor %q...", actorID) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ @@ -148,7 +167,9 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc } // - // Repoint the suspended actor at template B. + // Step 5: Repoint the suspended actor at template B. The update is a pure + // spec change: the actor stays SUSPENDED and keeps its latest snapshot, + // which still records template A as the one it was taken under. // t.Logf("Repointing Actor %q at template %q...", actorID, nameB) updated, err := clients.SubstrateAPI.UpdateActor(ctx, &ateapipb.UpdateActorRequest{Actor: &ateapipb.Actor{ @@ -166,8 +187,9 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc } // - // Resume under template B: the guest cold-boots from B (memory counter - // resets) while the durable dir carries over (file counter continues, + // Step 6: Resume under template B: the guest resumes from B's golden + // snapshot (whose memory counter is zero — the golden actor served no + // requests) while the durable dir carries over (file counter continues, // and B's --validate-existing-file-path reads A's file back). // t.Logf("Resuming Actor %q under template B...", actorID) @@ -187,8 +209,56 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc t.Errorf("[after template update] expected %q (template B validating the preserved file), got response: %s", want, resp) } - // A second suspend closes the loop: the resume under B stamped B as the - // sprint's template, and the suspend preserves it. + // The memory counter alone cannot tell B's golden from a cold boot — both + // answer the first call with 1. The boot uuid can: it exists only in + // guest memory, so a fresh actor resumed from template B (which rides + // B's golden, exactly like the resume under A above rode A's) reveals + // B's golden's uuid, and the repointed actor must report the very same + // value. A cold boot generates a new uuid and the A lineage carries A's + // golden's, so only a restore of B's golden memory image can match. + repointedBootUUID := parseBootUUID(t, resp) + if repointedBootUUID == goldenABootUUID { + t.Errorf("[after template update] boot uuid %q equals template A's golden: the replaced template's guest state survived the repoint", repointedBootUUID) + } + + // + // Step 7: Learn B's golden's boot uuid from a reference actor — a fresh + // actor created from B whose first resume rides B's golden — and require + // the repointed actor to have reported the very same value. + // + refActorID := "update-ref-" + nsObj.Name + t.Logf("Creating reference Actor %q from template B to observe B's golden boot uuid...", refActorID) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: refActorID}, + ActorTemplate: refB, + }}); err != nil { + t.Fatalf("failed to create reference Actor: %v", err) + } + defer func() { + clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: refActorID}, + }) + }() + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: refActorID}, + }); err != nil { + t.Fatalf("failed to resume reference Actor: %v", err) + } + waitForActorState(ctx, t, clients, refActorID, ateapipb.ActorState_ACTOR_STATE_RUNNING) + refResp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: refActorID}) + if err != nil { + t.Fatalf("failed to call reference actor: %v", err) + } + goldenBBootUUID := parseBootUUID(t, refResp) + if repointedBootUUID != goldenBBootUUID { + t.Errorf("[after template update] boot uuid = %q, want template B's golden %q: the repointed actor did not restore B's golden snapshot (a cold boot regenerates the uuid)", repointedBootUUID, goldenBBootUUID) + } + + // + // Step 8: Suspend again: the resume under B stamped B as the sprint's + // template, and the suspend preserves it. From here on the actor's latest + // snapshot and its template agree, so the repoint path no longer applies. + // t.Logf("Suspending Actor %q again...", actorID) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -205,6 +275,59 @@ func runUpdateTemplateTestCase(t *testing.T, onCommit ateapipb.SnapshotContentSc if got, want := suspended.GetStatus().GetCurrentActorTemplateUid(), createdB.GetMetadata().GetUid(); got != want { t.Errorf("re-suspended Actor current_actor_template_uid = %q, want template B's %q", got, want) } + + // + // Step 9: Resume once more. The latest snapshot was taken under B, so + // this resume no longer takes the repoint path: at Full scope it + // restores the actor's own snapshot, at Data scope it is a plain Golden + // data resume (B's golden + the actor's data) under the onResume policy. + // + t.Logf("Resuming Actor %q again under template B...", actorID) + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, + }); err != nil { + t.Fatalf("failed to resume Actor after re-suspend: %v", err) + } + waitForActorState(ctx, t, clients, actorID, ateapipb.ActorState_ACTOR_STATE_RUNNING) + + resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorID}) + if err != nil { + t.Fatalf("failed to call actor after second resume under B: %v", err) + } + // The durable dir keeps counting either way: this is the fourth call over + // the actor's lifetime. The memory counter is what tells the sources + // apart at Full scope: the suspend snapshot's guest served one call under + // B, so restoring it answers 2, while B's golden (memory counter zero) + // would answer 1 — the same as the repointed resume did in step 6. At + // Data scope the suspend snapshot carries no guest state and the resume + // rides B's golden again, so the memory counter starts over. + wantMemory := 1 + if onCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL { + wantMemory = 2 + } + validateCounterResponse(t, resp, "after second resume under B", wantMemory, 4) + if onCommit == ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL { + // The restored memory must also still descend from B's golden boot: + // a cold boot would regenerate the uuid. + if got := parseBootUUID(t, resp); got != goldenBBootUUID { + t.Errorf("[after second resume under B] boot uuid = %q, want template B's golden %q: the full snapshot's guest state was not restored", got, goldenBBootUUID) + } + } +} + +var bootUUIDPattern = regexp.MustCompile(`boot uuid: (\S+)`) + +// parseBootUUID extracts the boot uuid the counter workload generates at +// process startup. It lives only in guest memory, so a restore carries it +// over while a cold boot regenerates it: two sprints report the same uuid +// exactly when their memory descends from the same boot. +func parseBootUUID(t *testing.T, resp string) string { + t.Helper() + m := bootUUIDPattern.FindStringSubmatch(resp) + if m == nil { + t.Fatalf("response carries no boot uuid: %s", resp) + } + return m[1] } // createUpdateTestTemplate creates a per-test WorkerPool plus a substrate @@ -222,6 +345,9 @@ func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl StorageLocation: "gs://" + bucket + "/ate-demo-" + name, OnPause: onCommit, OnCommit: onCommit, + // A Data-scope capture resumes data-only under the default + // ColdBoot policy; the golden ride under test needs Golden. + OnResume: &ateapipb.OnResumeConfig{FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN}, }, Modify: modify, })