Make SandboxConfig spec immutable; mark defaults via annotation

.golangci-kal.yaml

@@ -96,7 +96,7 @@ linters:
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
         text: '^requiredfields: .*\bAssetFile\.(SHA256|URL)\b'
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
-        text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|Default|PauseImage|SandboxClass)\b'
+        text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|PauseImage|SandboxClass)\b'
       - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go'
         text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b'
       - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go'

cmd/ateapi/internal/controlapi/functionaltest/common_test.go

@@ -373,10 +373,12 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
 	t.Helper()
 	const name = "gvisor-default"
 	sc := &atev1alpha1.SandboxConfig{
-		ObjectMeta: metav1.ObjectMeta{Name: name},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: map[string]string{atev1alpha1.IsDefaultAnnotation: "true"},
+		},
 		Spec: atev1alpha1.SandboxConfigSpec{
 			SandboxClass: atev1alpha1.SandboxClassGvisor,
-			Default:      true,
 			PauseImage:   testPauseImage,
 			Assets: map[string]map[string]atev1alpha1.AssetFile{
 				"amd64": {"runsc": {

cmd/ateapi/internal/controlapi/sandbox_assets.go

@@ -63,26 +63,38 @@ func resolveSandboxAssets(
 	return sandboxAssetsProto(class, sc), nil
 }
 
-// defaultSandboxConfig returns the single SandboxConfig marked Default for the
-// given class, erroring if there are zero or more than one.
+// defaultSandboxConfig returns the SandboxConfig marked default (via the
+// IsDefaultAnnotation) for the given class. When several are marked, the most
+// recently created wins (as with StorageClass), so a new default can be
+// created before the old one is unmarked.
 func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev1alpha1.SandboxClass) (*atev1alpha1.SandboxConfig, error) {
 	all, err := lister.List(labels.Everything())
 	if err != nil {
 		return nil, fmt.Errorf("while listing SandboxConfigs: %w", err)
 	}
-	var match *atev1alpha1.SandboxConfig
+	var newest *atev1alpha1.SandboxConfig
 	for _, sc := range all {
-		if sc.Spec.SandboxClass == class && sc.Spec.Default {
-			if match != nil {
-				return nil, fmt.Errorf("multiple default SandboxConfigs for class %q (%q and %q)", class, match.Name, sc.Name)
-			}
-			match = sc
+		if sc.Spec.SandboxClass != class || !sc.IsDefault() {
+			continue
 		}
+		if newest == nil || defaultWinsOver(sc, newest) {
+			newest = sc
+		}
+	}
+	if newest == nil {
+		return nil, fmt.Errorf("no default SandboxConfig for class %q; annotate one with %s=\"true\" or name one via WorkerPool.spec.sandboxConfigName", class, atev1alpha1.IsDefaultAnnotation)
 	}
-	if match == nil {
-		return nil, fmt.Errorf("no default SandboxConfig for class %q; set one with spec.default=true or name one via WorkerPool.spec.sandboxConfigName", class)
+	return newest, nil
+}
+
+// defaultWinsOver reports whether a beats b as the class default: newer
+// creationTimestamp, smaller name on a tie (the same order StorageClass
+// default resolution uses).
+func defaultWinsOver(a, b *atev1alpha1.SandboxConfig) bool {
+	if !a.CreationTimestamp.Equal(&b.CreationTimestamp) {
+		return b.CreationTimestamp.Before(&a.CreationTimestamp)
 	}
-	return match, nil
+	return a.Name < b.Name
 }
 
 // sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet

cmd/ateapi/internal/controlapi/sandbox_assets_test.go

@@ -15,7 +15,9 @@
 package controlapi
 
 import (
+	"strings"
 	"testing"
+	"time"
 
 	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 	listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
@@ -57,10 +59,12 @@ func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) {
 		namedPause   = "gcr.io/gke-release/pause@sha256:named"
 	)
 	defaultConfig := &atev1alpha1.SandboxConfig{
-		ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "gvisor-default",
+			Annotations: map[string]string{atev1alpha1.IsDefaultAnnotation: "true"},
+		},
 		Spec: atev1alpha1.SandboxConfigSpec{
 			SandboxClass: atev1alpha1.SandboxClassGvisor,
-			Default:      true,
 			PauseImage:   defaultPause,
 			Assets:       testAssets(),
 		},
@@ -101,3 +105,91 @@ func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) {
 		})
 	}
 }
+
+// defaultAt builds a SandboxConfig with the given is-default annotation value
+// and creation time.
+func defaultAt(name string, class atev1alpha1.SandboxClass, annotation string, created time.Time) *atev1alpha1.SandboxConfig {
+	return &atev1alpha1.SandboxConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              name,
+			Annotations:       map[string]string{atev1alpha1.IsDefaultAnnotation: annotation},
+			CreationTimestamp: metav1.NewTime(created),
+		},
+		Spec: atev1alpha1.SandboxConfigSpec{
+			SandboxClass: class,
+			PauseImage:   "registry.k8s.io/pause@sha256:x",
+			Assets:       testAssets(),
+		},
+	}
+}
+
+// TestDefaultSandboxConfig pins the StorageClass-style default semantics:
+// only "true" counts, the newest default of the class wins, and none is an
+// error.
+func TestDefaultSandboxConfig(t *testing.T) {
+	t0 := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	tests := []struct {
+		name    string
+		configs []*atev1alpha1.SandboxConfig
+		want    string
+		wantErr string
+	}{{
+		name:    "single default",
+		configs: []*atev1alpha1.SandboxConfig{defaultAt("only", atev1alpha1.SandboxClassGvisor, "true", t0)},
+		want:    "only",
+	}, {
+		name: "most recently created wins",
+		configs: []*atev1alpha1.SandboxConfig{
+			defaultAt("old", atev1alpha1.SandboxClassGvisor, "true", t0),
+			defaultAt("new", atev1alpha1.SandboxClassGvisor, "true", t0.Add(time.Hour)),
+		},
+		want: "new",
+	}, {
+		name: "timestamp tie breaks by smaller name",
+		configs: []*atev1alpha1.SandboxConfig{
+			defaultAt("bbb", atev1alpha1.SandboxClassGvisor, "true", t0),
+			defaultAt("aaa", atev1alpha1.SandboxClassGvisor, "true", t0),
+		},
+		want: "aaa",
+	}, {
+		name: "other classes' defaults are ignored",
+		configs: []*atev1alpha1.SandboxConfig{
+			defaultAt("gv", atev1alpha1.SandboxClassGvisor, "true", t0),
+			defaultAt("mv", atev1alpha1.SandboxClassMicroVM, "true", t0.Add(time.Hour)),
+		},
+		want: "gv",
+	}, {
+		name: "only the exact value true counts",
+		configs: []*atev1alpha1.SandboxConfig{
+			defaultAt("upper", atev1alpha1.SandboxClassGvisor, "True", t0.Add(time.Hour)),
+			defaultAt("one", atev1alpha1.SandboxClassGvisor, "1", t0.Add(2*time.Hour)),
+			defaultAt("lower", atev1alpha1.SandboxClassGvisor, "true", t0),
+		},
+		want: "lower",
+	}, {
+		name: "no default is an error",
+		configs: []*atev1alpha1.SandboxConfig{
+			defaultAt("off", atev1alpha1.SandboxClassGvisor, "false", t0),
+		},
+		wantErr: "no default SandboxConfig",
+	}}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, configLister := listersFor(t, nil, tt.configs)
+			got, err := defaultSandboxConfig(configLister, atev1alpha1.SandboxClassGvisor)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("defaultSandboxConfig() error = %v, want it to contain %q", err, tt.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("defaultSandboxConfig() error: %v", err)
+			}
+			if got.Name != tt.want {
+				t.Errorf("defaultSandboxConfig() = %q, want %q", got.Name, tt.want)
+			}
+		})
+	}
+}

cmd/atelet/sandbox_prewarm.go

@@ -139,7 +139,7 @@ func startSandboxAssetPrewarm(ctx context.Context, informer cache.SharedIndexInf
 }
 
 // skipConfig reports whether this node has nothing to prewarm for cfg,
-// logging why. Both enqueue and process apply it: sandboxClass is mutable,
+// logging why. Both enqueue and process apply it: sandboxClass is immutable,
 // so the class observed at enqueue time can be stale by the time the worker
 // resolves the name after jitter or backoff, and the gate must hold for the
 // revision actually prewarmed.

docs/api-guide.md

@@ -360,13 +360,16 @@ Two consequences worth planning for:
 
 This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place.
 
+A `SandboxConfig`'s `spec` is **immutable**: snapshots pin restores to the exact binaries a sandbox booted with, so editing a config in place would strand every snapshot recorded against it. To roll to a new release, create a new `SandboxConfig` and mark it default or point the `WorkerPool` at it; only `metadata` stays mutable.
+
+The cluster default per class is marked with the `sandboxconfig.ate.dev/is-default: "true"` annotation (as with `StorageClass`). If several configs of one class carry it, the most recently created wins, so a new default can be created before the old one is unmarked. Values other than `"true"` are ignored.
+
 ### Specification (`SandboxConfigSpec`)
 
 | Field | Type | Description |
 | :--- | :--- | :--- |
 | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. |
 | `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. |
-| `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. |
 | `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. |
 
 A default cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor pools work out of the box.
@@ -378,9 +381,10 @@ apiVersion: ate.dev/v1alpha1
 kind: SandboxConfig
 metadata:
   name: gvisor-default
+  annotations:
+    sandboxconfig.ate.dev/is-default: "true"
 spec:
   sandboxClass: gvisor
-  default: true
   pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"
   assets:
     amd64:

manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml

@@ -34,9 +34,9 @@ spec:
     - jsonPath: .spec.sandboxClass
       name: Class
       type: string
-    - jsonPath: .spec.default
+    - jsonPath: .metadata.annotations['sandboxconfig\.ate\.dev/is-default']
       name: Default
-      type: boolean
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -45,8 +45,9 @@ spec:
       openAPIV3Schema:
         description: |-
           SandboxConfig is cluster-scoped configuration describing the sandbox binaries
-          for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools
-          and decouples sandbox binary selection from ActorTemplate.
+          for a sandbox runtime family. It is referenced (or defaulted, via the
+          IsDefaultAnnotation) by WorkerPools and decouples sandbox binary selection
+          from ActorTemplate.
         properties:
           apiVersion:
             description: |-
@@ -66,7 +67,10 @@ spec:
           metadata:
             type: object
           spec:
-            description: spec defines the desired state of SandboxConfig
+            description: |-
+              spec defines the desired state of SandboxConfig. It is immutable —
+              snapshots pin restores to the exact binaries a config named — so roll
+              to new assets by creating a new SandboxConfig.
             properties:
               assets:
                 additionalProperties:
@@ -104,13 +108,6 @@ spec:
                   intentionally generic; per-class requirements are enforced by a
                   ValidatingAdmissionPolicy.
                 type: object
-              default:
-                description: |-
-                  Default marks this SandboxConfig as the cluster-wide default for its
-                  SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to
-                  the default config for its SandboxClass. At most one default is expected
-                  per SandboxClass.
-                type: boolean
               pauseImage:
                 description: |-
                   PauseImage is the container image used as the root sandbox container.
@@ -142,6 +139,10 @@ spec:
             - pauseImage
             - sandboxClass
             type: object
+            x-kubernetes-validations:
+            - message: SandboxConfig spec is immutable; create a new SandboxConfig
+                and mark it default
+              rule: self == oldSelf
         required:
         - spec
         type: object

manifests/ate-install/sandboxconfig-gvisor.yaml

@@ -16,16 +16,19 @@
 # WorkerPool with sandboxClass gvisor (the default) and no explicit
 # sandboxConfigName resolves to this. atelet fetches the gVisor release tarball
 # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to
-# it) matching the worker node's architecture and extracts it locally. To pin a
-# different release, edit the assets below or create another SandboxConfig and
-# name it from the WorkerPool.
+# it) matching the worker node's architecture and extracts it locally.
+#
+# The spec is immutable. To roll to a different release, create a new
+# SandboxConfig marked default with the annotation below (the most recently
+# created default of a class wins) or name it from the WorkerPool.
 apiVersion: ate.dev/v1alpha1
 kind: SandboxConfig
 metadata:
   name: gvisor-default
+  annotations:
+    sandboxconfig.ate.dev/is-default: "true"
 spec:
   sandboxClass: gvisor
-  default: true
   # The root sandbox container's image. On GCP, prefer the in-project mirror
   # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da.
   pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4"

manifests/microvm/sandboxconfig-microvm.yaml.tmpl

@@ -14,15 +14,17 @@
 
 # Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox
 # class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by
-# hack/install-ate.sh --deploy-ate-system and marked default:true), this is
-# opt-in: apply via hack/install-microvm-deps.sh --install after staging the
-# asset set (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh).
+# hack/install-ate.sh --deploy-ate-system and marked default via the
+# sandboxconfig.ate.dev/is-default annotation), this is opt-in: apply via
+# hack/install-microvm-deps.sh --install after staging the asset set
+# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh).
 #
-# It is deliberately NOT marked default:true. A dirty teardown could leave this
-# CR behind, and if it were the class default a subsequent WorkerPool that
-# omitted sandboxConfigName would silently resolve to the stale config. Making
-# every microvm WorkerPool name it explicitly (sandboxConfigName: microvm) makes
-# a missing/stale config fail loudly instead.
+# It deliberately does NOT carry the is-default annotation. A dirty teardown
+# could leave this CR behind, and if it were the class default a subsequent
+# WorkerPool that omitted sandboxConfigName would silently resolve to the stale
+# config. Making every microvm WorkerPool name it explicitly
+# (sandboxConfigName: microvm) makes a missing/stale config fail loudly
+# instead.
 #
 # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs,
 # base configuration.toml) are FETCHED at runtime from the cluster object store