diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 1607c9ebbe..94297cf114 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -96,7 +96,7 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^requiredfields: .*\bAssetFile\.(SHA256|URL)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|Default|PauseImage|SandboxClass)\b' + text: '^(nomaps|optionalfields|requiredfields): .*\bSandboxConfigSpec\.(Assets|PauseImage|SandboxClass)\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 9a8f44bfa9..dff92b603a 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -373,10 +373,12 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { t.Helper() const name = "gvisor-default" sc := &atev1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: name}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{atev1alpha1.IsDefaultAnnotation: "true"}, + }, Spec: atev1alpha1.SandboxConfigSpec{ SandboxClass: atev1alpha1.SandboxClassGvisor, - Default: true, PauseImage: testPauseImage, Assets: map[string]map[string]atev1alpha1.AssetFile{ "amd64": {"runsc": { diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index a564582592..76004985e9 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -63,26 +63,38 @@ func resolveSandboxAssets( return sandboxAssetsProto(class, sc), nil } -// defaultSandboxConfig returns the single SandboxConfig marked Default for the -// given class, erroring if there are zero or more than one. +// defaultSandboxConfig returns the SandboxConfig marked default (via the +// IsDefaultAnnotation) for the given class. When several are marked, the most +// recently created wins (as with StorageClass), so a new default can be +// created before the old one is unmarked. func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev1alpha1.SandboxClass) (*atev1alpha1.SandboxConfig, error) { all, err := lister.List(labels.Everything()) if err != nil { return nil, fmt.Errorf("while listing SandboxConfigs: %w", err) } - var match *atev1alpha1.SandboxConfig + var newest *atev1alpha1.SandboxConfig for _, sc := range all { - if sc.Spec.SandboxClass == class && sc.Spec.Default { - if match != nil { - return nil, fmt.Errorf("multiple default SandboxConfigs for class %q (%q and %q)", class, match.Name, sc.Name) - } - match = sc + if sc.Spec.SandboxClass != class || !sc.IsDefault() { + continue } + if newest == nil || defaultWinsOver(sc, newest) { + newest = sc + } + } + if newest == nil { + return nil, fmt.Errorf("no default SandboxConfig for class %q; annotate one with %s=\"true\" or name one via WorkerPool.spec.sandboxConfigName", class, atev1alpha1.IsDefaultAnnotation) } - if match == nil { - return nil, fmt.Errorf("no default SandboxConfig for class %q; set one with spec.default=true or name one via WorkerPool.spec.sandboxConfigName", class) + return newest, nil +} + +// defaultWinsOver reports whether a beats b as the class default: newer +// creationTimestamp, smaller name on a tie (the same order StorageClass +// default resolution uses). +func defaultWinsOver(a, b *atev1alpha1.SandboxConfig) bool { + if !a.CreationTimestamp.Equal(&b.CreationTimestamp) { + return b.CreationTimestamp.Before(&a.CreationTimestamp) } - return match, nil + return a.Name < b.Name } // sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go index a20bd35a18..aa2bd11e38 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -15,7 +15,9 @@ package controlapi import ( + "strings" "testing" + "time" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" @@ -57,10 +59,12 @@ func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) { namedPause = "gcr.io/gke-release/pause@sha256:named" ) defaultConfig := &atev1alpha1.SandboxConfig{ - ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "gvisor-default", + Annotations: map[string]string{atev1alpha1.IsDefaultAnnotation: "true"}, + }, Spec: atev1alpha1.SandboxConfigSpec{ SandboxClass: atev1alpha1.SandboxClassGvisor, - Default: true, PauseImage: defaultPause, Assets: testAssets(), }, @@ -101,3 +105,91 @@ func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) { }) } } + +// defaultAt builds a SandboxConfig with the given is-default annotation value +// and creation time. +func defaultAt(name string, class atev1alpha1.SandboxClass, annotation string, created time.Time) *atev1alpha1.SandboxConfig { + return &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{atev1alpha1.IsDefaultAnnotation: annotation}, + CreationTimestamp: metav1.NewTime(created), + }, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: class, + PauseImage: "registry.k8s.io/pause@sha256:x", + Assets: testAssets(), + }, + } +} + +// TestDefaultSandboxConfig pins the StorageClass-style default semantics: +// only "true" counts, the newest default of the class wins, and none is an +// error. +func TestDefaultSandboxConfig(t *testing.T) { + t0 := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + tests := []struct { + name string + configs []*atev1alpha1.SandboxConfig + want string + wantErr string + }{{ + name: "single default", + configs: []*atev1alpha1.SandboxConfig{defaultAt("only", atev1alpha1.SandboxClassGvisor, "true", t0)}, + want: "only", + }, { + name: "most recently created wins", + configs: []*atev1alpha1.SandboxConfig{ + defaultAt("old", atev1alpha1.SandboxClassGvisor, "true", t0), + defaultAt("new", atev1alpha1.SandboxClassGvisor, "true", t0.Add(time.Hour)), + }, + want: "new", + }, { + name: "timestamp tie breaks by smaller name", + configs: []*atev1alpha1.SandboxConfig{ + defaultAt("bbb", atev1alpha1.SandboxClassGvisor, "true", t0), + defaultAt("aaa", atev1alpha1.SandboxClassGvisor, "true", t0), + }, + want: "aaa", + }, { + name: "other classes' defaults are ignored", + configs: []*atev1alpha1.SandboxConfig{ + defaultAt("gv", atev1alpha1.SandboxClassGvisor, "true", t0), + defaultAt("mv", atev1alpha1.SandboxClassMicroVM, "true", t0.Add(time.Hour)), + }, + want: "gv", + }, { + name: "only the exact value true counts", + configs: []*atev1alpha1.SandboxConfig{ + defaultAt("upper", atev1alpha1.SandboxClassGvisor, "True", t0.Add(time.Hour)), + defaultAt("one", atev1alpha1.SandboxClassGvisor, "1", t0.Add(2*time.Hour)), + defaultAt("lower", atev1alpha1.SandboxClassGvisor, "true", t0), + }, + want: "lower", + }, { + name: "no default is an error", + configs: []*atev1alpha1.SandboxConfig{ + defaultAt("off", atev1alpha1.SandboxClassGvisor, "false", t0), + }, + wantErr: "no default SandboxConfig", + }} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, configLister := listersFor(t, nil, tt.configs) + got, err := defaultSandboxConfig(configLister, atev1alpha1.SandboxClassGvisor) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("defaultSandboxConfig() error = %v, want it to contain %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("defaultSandboxConfig() error: %v", err) + } + if got.Name != tt.want { + t.Errorf("defaultSandboxConfig() = %q, want %q", got.Name, tt.want) + } + }) + } +} diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go index 5b360f7b19..0153f48126 100644 --- a/cmd/atelet/sandbox_prewarm.go +++ b/cmd/atelet/sandbox_prewarm.go @@ -139,7 +139,7 @@ func startSandboxAssetPrewarm(ctx context.Context, informer cache.SharedIndexInf } // skipConfig reports whether this node has nothing to prewarm for cfg, -// logging why. Both enqueue and process apply it: sandboxClass is mutable, +// logging why. Both enqueue and process apply it: sandboxClass is immutable, // so the class observed at enqueue time can be stale by the time the worker // resolves the name after jitter or backoff, and the gate must hold for the // revision actually prewarmed. diff --git a/docs/api-guide.md b/docs/api-guide.md index 5a5302ca8c..c3a874dac4 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -360,13 +360,16 @@ Two consequences worth planning for: This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. +A `SandboxConfig`'s `spec` is **immutable**: snapshots pin restores to the exact binaries a sandbox booted with, so editing a config in place would strand every snapshot recorded against it. To roll to a new release, create a new `SandboxConfig` and mark it default or point the `WorkerPool` at it; only `metadata` stays mutable. + +The cluster default per class is marked with the `sandboxconfig.ate.dev/is-default: "true"` annotation (as with `StorageClass`). If several configs of one class carry it, the most recently created wins, so a new default can be created before the old one is unmarked. Values other than `"true"` are ignored. + ### Specification (`SandboxConfigSpec`) | Field | Type | Description | | :--- | :--- | :--- | | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | | `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | -| `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. | | `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. | A default cluster-wide gVisor `SandboxConfig` (`gvisor-default`) is installed with the platform, so gVisor pools work out of the box. @@ -378,9 +381,10 @@ apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-default: "true" spec: sandboxClass: gvisor - default: true pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 6a7ee9871c..1fd8258f9c 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -34,9 +34,9 @@ spec: - jsonPath: .spec.sandboxClass name: Class type: string - - jsonPath: .spec.default + - jsonPath: .metadata.annotations['sandboxconfig\.ate\.dev/is-default'] name: Default - type: boolean + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -45,8 +45,9 @@ spec: openAPIV3Schema: description: |- SandboxConfig is cluster-scoped configuration describing the sandbox binaries - for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools - and decouples sandbox binary selection from ActorTemplate. + for a sandbox runtime family. It is referenced (or defaulted, via the + IsDefaultAnnotation) by WorkerPools and decouples sandbox binary selection + from ActorTemplate. properties: apiVersion: description: |- @@ -66,7 +67,10 @@ spec: metadata: type: object spec: - description: spec defines the desired state of SandboxConfig + description: |- + spec defines the desired state of SandboxConfig. It is immutable — + snapshots pin restores to the exact binaries a config named — so roll + to new assets by creating a new SandboxConfig. properties: assets: additionalProperties: @@ -104,13 +108,6 @@ spec: intentionally generic; per-class requirements are enforced by a ValidatingAdmissionPolicy. type: object - default: - description: |- - Default marks this SandboxConfig as the cluster-wide default for its - SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to - the default config for its SandboxClass. At most one default is expected - per SandboxClass. - type: boolean pauseImage: description: |- PauseImage is the container image used as the root sandbox container. @@ -142,6 +139,10 @@ spec: - pauseImage - sandboxClass type: object + x-kubernetes-validations: + - message: SandboxConfig spec is immutable; create a new SandboxConfig + and mark it default + rule: self == oldSelf required: - spec type: object diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 4fb9d0f3d8..f33b66d89f 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -16,16 +16,19 @@ # WorkerPool with sandboxClass gvisor (the default) and no explicit # sandboxConfigName resolves to this. atelet fetches the gVisor release tarball # (gvisor.tar.zstd: runsc plus the gvisor-bin/ helpers runsc requires next to -# it) matching the worker node's architecture and extracts it locally. To pin a -# different release, edit the assets below or create another SandboxConfig and -# name it from the WorkerPool. +# it) matching the worker node's architecture and extracts it locally. +# +# The spec is immutable. To roll to a different release, create a new +# SandboxConfig marked default with the annotation below (the most recently +# created default of a class wins) or name it from the WorkerPool. apiVersion: ate.dev/v1alpha1 kind: SandboxConfig metadata: name: gvisor-default + annotations: + sandboxconfig.ate.dev/is-default: "true" spec: sandboxClass: gvisor - default: true # The root sandbox container's image. On GCP, prefer the in-project mirror # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 7501c20d8f..2dc072ef2f 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -14,15 +14,17 @@ # Cluster-wide SandboxConfig for the micro-VM (kata + cloud-hypervisor) sandbox # class. Unlike sandboxconfig-gvisor.yaml (applied unconditionally by -# hack/install-ate.sh --deploy-ate-system and marked default:true), this is -# opt-in: apply via hack/install-microvm-deps.sh --install after staging the -# asset set (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). +# hack/install-ate.sh --deploy-ate-system and marked default via the +# sandboxconfig.ate.dev/is-default annotation), this is opt-in: apply via +# hack/install-microvm-deps.sh --install after staging the asset set +# (assemble.sh + stage-to-gcs.sh / stage-to-rustfs.sh). # -# It is deliberately NOT marked default:true. A dirty teardown could leave this -# CR behind, and if it were the class default a subsequent WorkerPool that -# omitted sandboxConfigName would silently resolve to the stale config. Making -# every microvm WorkerPool name it explicitly (sandboxConfigName: microvm) makes -# a missing/stale config fail loudly instead. +# It deliberately does NOT carry the is-default annotation. A dirty teardown +# could leave this CR behind, and if it were the class default a subsequent +# WorkerPool that omitted sandboxConfigName would silently resolve to the stale +# config. Making every microvm WorkerPool name it explicitly +# (sandboxConfigName: microvm) makes a missing/stale config fail loudly +# instead. # # The sandbox binaries (cloud-hypervisor, virtiofsd, guest kernel, guest rootfs, # base configuration.toml) are FETCHED at runtime from the cluster object store diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 81cf44a589..1b4bfcfda3 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -30,6 +30,12 @@ const ( SandboxClassMicroVM SandboxClass = "microvm" ) +// IsDefaultAnnotation set to "true" marks a SandboxConfig as the cluster-wide +// default for its SandboxClass; other values are ignored. If several configs +// of a class carry it, the most recently created wins (as with StorageClass). +// An annotation rather than a spec field because the spec is immutable. +const IsDefaultAnnotation = "sandboxconfig.ate.dev/is-default" + // AssetFile is one content-addressed file that atelet fetches for a sandbox // runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). type AssetFile struct { @@ -59,14 +65,6 @@ type SandboxConfigSpec struct { // +kubebuilder:default=gvisor SandboxClass SandboxClass `json:"sandboxClass"` - // Default marks this SandboxConfig as the cluster-wide default for its - // SandboxClass. A WorkerPool with no explicit SandboxConfigName resolves to - // the default config for its SandboxClass. At most one default is expected - // per SandboxClass. - // - // +optional - Default bool `json:"default,omitempty"` - // PauseImage is the container image used as the root sandbox container. // It holds the sandbox's namespaces and runs no workload code, so it is an // implementation detail of the sandbox rather than something actor authors @@ -98,8 +96,9 @@ type SandboxConfigSpec struct { } // SandboxConfig is cluster-scoped configuration describing the sandbox binaries -// for a sandbox runtime family. It is referenced (or defaulted) by WorkerPools -// and decouples sandbox binary selection from ActorTemplate. +// for a sandbox runtime family. It is referenced (or defaulted, via the +// IsDefaultAnnotation) by WorkerPools and decouples sandbox binary selection +// from ActorTemplate. // // +genclient // +genclient:nonNamespaced @@ -107,7 +106,7 @@ type SandboxConfigSpec struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=sandboxconfig // +kubebuilder:printcolumn:name="Class",type=string,JSONPath=`.spec.sandboxClass` -// +kubebuilder:printcolumn:name="Default",type=boolean,JSONPath=`.spec.default` +// +kubebuilder:printcolumn:name="Default",type=string,JSONPath=`.metadata.annotations['sandboxconfig\.ate\.dev/is-default']` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type SandboxConfig struct { metav1.TypeMeta `json:",inline"` @@ -116,11 +115,21 @@ type SandboxConfig struct { // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - // spec defines the desired state of SandboxConfig + // spec defines the desired state of SandboxConfig. It is immutable — + // snapshots pin restores to the exact binaries a config named — so roll + // to new assets by creating a new SandboxConfig. + // // +required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="SandboxConfig spec is immutable; create a new SandboxConfig and mark it default" Spec SandboxConfigSpec `json:"spec"` } +// IsDefault reports whether this config is marked as the cluster-wide default +// for its SandboxClass. +func (sc *SandboxConfig) IsDefault() bool { + return sc.Annotations[IsDefaultAnnotation] == "true" +} + // SandboxConfigList contains a list of SandboxConfigs. // +kubebuilder:object:generate=true // +kubebuilder:object:root=true diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 374edf966d..9d8919a2d8 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -225,3 +225,64 @@ func TestSandboxConfigValidation(t *testing.T) { }) } } + +// TestSandboxConfigSpecImmutable pins that the spec rejects any change after +// creation while metadata (notably the is-default annotation) stays mutable. +func TestSandboxConfigSpecImmutable(t *testing.T) { + ctx := t.Context() + + tests := []struct { + name string + mutate func(sc *SandboxConfig) + wantErr bool + }{{ + name: "changing an asset URL is rejected", + mutate: func(sc *SandboxConfig) { + sc.Spec.Assets["amd64"]["gvisor"] = AssetFile{URL: "gs://bucket/other", SHA256: validSHA256} + }, + wantErr: true, + }, { + name: "changing the pause image is rejected", + mutate: func(sc *SandboxConfig) { sc.Spec.PauseImage = "registry.k8s.io/pause:3.11@sha256:" + validSHA256 }, + wantErr: true, + }, { + name: "adding an architecture is rejected", + mutate: func(sc *SandboxConfig) { sc.Spec.Assets["arm64"] = map[string]AssetFile{"gvisor": gvisorAsset()} }, + wantErr: true, + }, { + name: "no-op update passes", + mutate: func(sc *SandboxConfig) {}, + }, { + name: "flipping the is-default annotation passes", + mutate: func(sc *SandboxConfig) { sc.Annotations = map[string]string{IsDefaultAnnotation: "true"} }, + }, { + name: "adding a label passes", + mutate: func(sc *SandboxConfig) { sc.Labels = map[string]string{"team": "substrate"} }, + }} + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := sandboxConfig(fmt.Sprintf("immutable-%d", i), SandboxClassGvisor, + map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}) + if err := k8sClient.Create(ctx, sc); err != nil { + t.Fatalf("Create() error: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, sc, &client.DeleteOptions{}) }) + + tt.mutate(sc) + err := k8sClient.Update(ctx, sc) + if tt.wantErr { + if err == nil { + t.Fatal("Update() succeeded, want denied") + } + if !strings.Contains(err.Error(), "immutable") { + t.Errorf("Update() error = %q, want it to contain %q", err.Error(), "immutable") + } + return + } + if err != nil { + t.Fatalf("Update() unexpected error: %v", err) + } + }) + } +}