From 3d089c7bbf252173280fc88e01fe56e7c3a68214 Mon Sep 17 00:00:00 2001 From: jeffluoo Date: Fri, 4 Sep 2026 17:10:10 +0000 Subject: [PATCH 1/4] metrics: report a worker with no sandbox class as unknown Control.CreateWorker does not validate Worker.sandbox_class, so a client can register a worker carrying an empty one, and any worker written before this keeps it. RegisterWorkerCount tallied that raw value, emitting ate.workerpool.workers with ate.sandbox.class="" -- not a member of the attribute's registry vocabulary -- next to the pool's seeded series at 0. Normalize the worker's class with ateattr.NormalizeSandboxClass, so an empty class reports as unknown. That is the rule the same helper already applies to this attribute on the atelet side, so the codebase keeps one rule rather than two. The worker is deliberately not folded into the series of the pool it names. Scheduling matches sandbox_class exactly, so a worker with no class can host no actor; counting it as pool capacity would inflate the idle state and silence an idle == 0 alert while every resume fails for want of a worker. Under unknown it stays visible and alertable, and the pool's own counts stay truthful. The empty class enters through the worker record, not the WorkerPool CR: the CRD defaults sandboxClass to gvisor and rejects "" by enum. Validating Worker.sandbox_class at the API boundary would stop the empty state existing at all; that is a scheduling fix, and a separate change. --- cmd/ateapi/internal/controlapi/metrics.go | 7 +- .../internal/controlapi/metrics_test.go | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index be46f3b80b..e5d5fdf2d1 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -102,7 +102,12 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, if w.GetStatus().GetAllocated().GetActors() > 0 { state = ateattr.WorkerStateAssigned } - tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, w.GetSandboxClass()}]++ + // CreateWorker does not validate the class, thus a worker can have an + // empty one. Report it as unknown, not as the pool's class: the + // scheduler puts no actor on a worker that offers no class, thus the + // pool's series must not count it as free capacity. + class := ateattr.NormalizeSandboxClass(w.GetSandboxClass()) + tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, class}]++ } for k, n := range tally { o.ObserveInt64(counter, n, metric.WithAttributes( diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 58df824131..0bcb1d5b02 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -400,3 +400,102 @@ func TestWorkerCountSeedsZeroForKnownPools(t *testing.T) { } } } + +// TestWorkerCountReportsEmptyClassWorkerAsUnknown asserts that workers registered +// through CreateWorker with no sandbox class report as unknown, not as an +// empty-string class and not as capacity of the pool they name. +func TestWorkerCountReportsEmptyClassWorkerAsUnknown(t *testing.T) { + pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { + return []*atev1alpha1.WorkerPool{ + workerPool("ns-1", "pool-empty", ""), + }, nil + } + workers := func() ([]*ateapipb.Worker, error) { + return []*ateapipb.Worker{ + worker("ns-1", "pool-empty", "", false), + worker("ns-1", "pool-empty", "", false), + worker("ns-1", "pool-empty", "", false), + }, nil + } + reader := newWorkerCountReader(t, workers, pools) + + sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) + got := seriesCounts(sum) + // The pool's own series stays at 0: none of the three can be scheduled, so + // an idle==0 alert on the gvisor capacity still fires, and the three stay + // visible under unknown. + want := map[series]int64{ + {"ns-1", "pool-empty", ateattr.WorkerStateIdle, "gvisor"}: 0, + {"ns-1", "pool-empty", ateattr.WorkerStateAssigned, "gvisor"}: 0, + {"ns-1", "pool-empty", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 3, + } + if len(got) != len(want) { + t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) + } + for k, v := range want { + if gv, ok := got[k]; !ok || gv != v { + t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) + } + } +} + +// TestWorkerCountMicroVMPoolWithEmptyClassWorker asserts that the same holds for +// a microvm pool: the empty-class worker reports as unknown, and no gvisor series +// appears for a pool that runs no gvisor. +func TestWorkerCountMicroVMPoolWithEmptyClassWorker(t *testing.T) { + pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { + return []*atev1alpha1.WorkerPool{ + workerPool("ns-1", "pool-micro", atev1alpha1.SandboxClassMicroVM), + }, nil + } + workers := func() ([]*ateapipb.Worker, error) { + return []*ateapipb.Worker{ + worker("ns-1", "pool-micro", "", false), + }, nil + } + reader := newWorkerCountReader(t, workers, pools) + + sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) + got := seriesCounts(sum) + want := map[series]int64{ + {"ns-1", "pool-micro", ateattr.WorkerStateIdle, "microvm"}: 0, + {"ns-1", "pool-micro", ateattr.WorkerStateAssigned, "microvm"}: 0, + {"ns-1", "pool-micro", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, + } + if len(got) != len(want) { + t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) + } + for k, v := range want { + if gv, ok := got[k]; !ok || gv != v { + t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) + } + } +} + +// TestWorkerCountOrphanWorkerWithEmptyClass asserts that an empty-class worker belonging +// to no known pool falls back to unknown rather than emitting an empty-string class. +func TestWorkerCountOrphanWorkerWithEmptyClass(t *testing.T) { + pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { + return nil, nil + } + workers := func() ([]*ateapipb.Worker, error) { + return []*ateapipb.Worker{ + worker("ns-1", "pool-orphan", "", false), + }, nil + } + reader := newWorkerCountReader(t, workers, pools) + + sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) + got := seriesCounts(sum) + want := map[series]int64{ + {"ns-1", "pool-orphan", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, + } + if len(got) != len(want) { + t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) + } + for k, v := range want { + if gv, ok := got[k]; !ok || gv != v { + t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) + } + } +} From 7ad23eeb121b28529ba88b884f2864bc580202d5 Mon Sep 17 00:00:00 2001 From: Jeff Luo Date: Tue, 8 Sep 2026 10:43:10 -0400 Subject: [PATCH 2/4] metrics: give ate.sandbox.class one normalization rule The class reaches the emitters from sources that nothing validates, and each emitter decided for itself what to do with a value outside the attribute's vocabulary. Only atelet normalized. So ate.actor.crashes and the ate.actor.lifecycle timers could carry ate.sandbox.class="", which the registry marks required and permits only gvisor, microvm and unknown. The crash counter has a source of its own, separate from the unvalidated worker record: releaseWorker returns an empty class when the worker is already gone or its assignment is already cleared. There the empty means the class could not be read, which is what unknown is for. Add ateattr.SandboxClassAttribute and stamp the attribute through it, so the rule lives in one place. recordSchedulerAssignment keeps its own behavior: it leaves the attribute off when the class is unknown, which its comment explains, and that is a deliberate choice rather than a missing bound. --- cmd/ateapi/internal/controlapi/metrics.go | 2 +- cmd/atelet/metrics.go | 2 +- internal/ateattr/ateattr.go | 10 +++++++++- internal/ateattr/ateattr_test.go | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index e5d5fdf2d1..e8e539515b 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -196,7 +196,7 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *ateapipb.ActorTemplate, s ass := actor.GetStatus().GetWorkerAssignment() attrs = append(attrs, ateattr.WorkerPoolAttributes(ass.GetWorkerNamespace(), ass.GetWorkerPool())...) if template != nil { - attrs = append(attrs, ateattr.SandboxClassKey.String(sandboxClassString(template.GetSandboxConfig().GetSandboxClass()))) + attrs = append(attrs, ateattr.SandboxClassAttribute(sandboxClassString(template.GetSandboxConfig().GetSandboxClass()))) } if snapshotKind != "" { attrs = append(attrs, ateattr.SnapshotKindKey.String(snapshotKind)) diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index 896c835f5b..b48815b253 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -98,7 +98,7 @@ func (o snapshotOp) attrs() []attribute.KeyValue { attrs = append(attrs, ateattr.SnapshotKindKey.String(o.kind)) } if o.sandboxClass != "" { - attrs = append(attrs, ateattr.SandboxClassKey.String(ateattr.NormalizeSandboxClass(o.sandboxClass))) + attrs = append(attrs, ateattr.SandboxClassAttribute(o.sandboxClass)) } return attrs } diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index 702bf1263b..b3abc7dc72 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -343,6 +343,14 @@ func NormalizeSandboxClass(class string) string { } } +// SandboxClassAttribute sets ate.sandbox.class. Use it in place of +// SandboxClassKey: no source of the class is validated, thus all emitters must +// apply the same limit. recordSchedulerAssignment is the one exception. It +// omits the attribute when the class is unknown. +func SandboxClassAttribute(class string) attribute.KeyValue { + return SandboxClassKey.String(NormalizeSandboxClass(class)) +} + // WorkerPoolAttributes returns the namespaced identity of a WorkerPool. A // WorkerPool is namespaced, so half the pair identifies no pool: either key // missing drops both, rather than emit an empty-string series that merges @@ -445,7 +453,7 @@ func ActorMetricAttributes(a *ateapipb.Actor, sandboxClass, operationName, reaso attrs := []attribute.KeyValue{ TemplateAtespaceKey.String(a.GetActorTemplate().GetAtespace()), TemplateNameKey.String(a.GetActorTemplate().GetName()), - SandboxClassKey.String(sandboxClass), + SandboxClassAttribute(sandboxClass), ActorOperationNameKey.String(operationName), } attrs = append(attrs, FailureAttributes(reason)...) diff --git a/internal/ateattr/ateattr_test.go b/internal/ateattr/ateattr_test.go index 0fd7bdb312..76abffb6b9 100644 --- a/internal/ateattr/ateattr_test.go +++ b/internal/ateattr/ateattr_test.go @@ -469,6 +469,24 @@ func TestActorMetricAttributes(t *testing.T) { assertAttrs(t, got, want) }) + // releaseWorker gives an empty class when the worker record is gone. + // CreateWorker can also store one. Empty is not a permitted value. + t.Run("empty sandbox class is normalized to unknown", func(t *testing.T) { + got := toMap(ActorMetricAttributes(actor, "", OperationResume, ReasonCorruptedAssignment)) + want := map[attribute.Key]any{ + TemplateAtespaceKey: "default", + TemplateNameKey: "counter-template", + WorkerPoolNamespaceKey: "ate-workers", + WorkerPoolNameKey: "default-pool", + SandboxClassKey: SandboxClassUnknown, + ActorOperationNameKey: OperationResume, + FailureReasonKey: ReasonCorruptedAssignment, + FailureDomainKey: FailureDomainInfrastructure, + } + + assertAttrs(t, got, want) + }) + t.Run("out of range operation name is normalized to unknown", func(t *testing.T) { got := toMap(ActorMetricAttributes(actor, "gvisor", "invalid_op", "")) want := map[attribute.Key]any{ From 526af33a0c88b67bb5d87b7052cd3cffd8482bf5 Mon Sep 17 00:00:00 2001 From: Jeff Luo Date: Tue, 8 Sep 2026 15:08:22 -0400 Subject: [PATCH 3/4] metrics: normalize the scheduler's class, and correct the helper's comment Review fixes. recordEligibleWorkers set ate.sandbox.class from the raw Constraints value. The class comes from sandboxClassString, which gives an empty string for an unspecified enum, thus the same gap the helper closes elsewhere. The attribute is required on ate.scheduler.eligible_workers. Its test asked for the class "kata" to prove a mismatch. That value cannot reach the scheduler: sandboxClassString gives only gvisor, microvm or empty. The test now asks for microvm against a gvisor fleet, which is the same mismatch with a class that the vocabulary permits. The helper's comment named one omit-site. snapshotOp.attrs is a second, and its own comment says so. Also fold the three worker-count tests into a table, and take the class values from the API constants. --- .../internal/controlapi/metrics_test.go | 161 ++++++++---------- cmd/ateapi/internal/scheduling/metrics.go | 2 +- .../internal/scheduling/scheduling_test.go | 6 +- internal/ateattr/ateattr.go | 10 +- 4 files changed, 77 insertions(+), 102 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index 0bcb1d5b02..51d812c207 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -401,101 +401,74 @@ func TestWorkerCountSeedsZeroForKnownPools(t *testing.T) { } } -// TestWorkerCountReportsEmptyClassWorkerAsUnknown asserts that workers registered -// through CreateWorker with no sandbox class report as unknown, not as an -// empty-string class and not as capacity of the pool they name. -func TestWorkerCountReportsEmptyClassWorkerAsUnknown(t *testing.T) { - pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { - return []*atev1alpha1.WorkerPool{ - workerPool("ns-1", "pool-empty", ""), - }, nil - } - workers := func() ([]*ateapipb.Worker, error) { - return []*ateapipb.Worker{ - worker("ns-1", "pool-empty", "", false), - worker("ns-1", "pool-empty", "", false), - worker("ns-1", "pool-empty", "", false), - }, nil - } - reader := newWorkerCountReader(t, workers, pools) - - sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) - got := seriesCounts(sum) - // The pool's own series stays at 0: none of the three can be scheduled, so - // an idle==0 alert on the gvisor capacity still fires, and the three stay - // visible under unknown. - want := map[series]int64{ - {"ns-1", "pool-empty", ateattr.WorkerStateIdle, "gvisor"}: 0, - {"ns-1", "pool-empty", ateattr.WorkerStateAssigned, "gvisor"}: 0, - {"ns-1", "pool-empty", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 3, - } - if len(got) != len(want) { - t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) - } - for k, v := range want { - if gv, ok := got[k]; !ok || gv != v { - t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) - } - } -} - -// TestWorkerCountMicroVMPoolWithEmptyClassWorker asserts that the same holds for -// a microvm pool: the empty-class worker reports as unknown, and no gvisor series -// appears for a pool that runs no gvisor. -func TestWorkerCountMicroVMPoolWithEmptyClassWorker(t *testing.T) { - pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { - return []*atev1alpha1.WorkerPool{ - workerPool("ns-1", "pool-micro", atev1alpha1.SandboxClassMicroVM), - }, nil - } - workers := func() ([]*ateapipb.Worker, error) { - return []*ateapipb.Worker{ - worker("ns-1", "pool-micro", "", false), - }, nil - } - reader := newWorkerCountReader(t, workers, pools) - - sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) - got := seriesCounts(sum) - want := map[series]int64{ - {"ns-1", "pool-micro", ateattr.WorkerStateIdle, "microvm"}: 0, - {"ns-1", "pool-micro", ateattr.WorkerStateAssigned, "microvm"}: 0, - {"ns-1", "pool-micro", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, - } - if len(got) != len(want) { - t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) - } - for k, v := range want { - if gv, ok := got[k]; !ok || gv != v { - t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) - } - } -} - -// TestWorkerCountOrphanWorkerWithEmptyClass asserts that an empty-class worker belonging -// to no known pool falls back to unknown rather than emitting an empty-string class. -func TestWorkerCountOrphanWorkerWithEmptyClass(t *testing.T) { - pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { - return nil, nil - } - workers := func() ([]*ateapipb.Worker, error) { - return []*ateapipb.Worker{ - worker("ns-1", "pool-orphan", "", false), - }, nil +// TestWorkerCountEmptyClassWorker covers the workers that CreateWorker accepted +// with no sandbox class. They report as unknown, not as an empty-string class, +// and not as capacity of the pool they name: the scheduler puts no actor on +// them, thus the pool's own series stays at 0 and an idle==0 alert on it still +// fires. +func TestWorkerCountEmptyClassWorker(t *testing.T) { + const ( + gvisor = string(atev1alpha1.SandboxClassGvisor) + microvm = string(atev1alpha1.SandboxClassMicroVM) + ) + tests := []struct { + name string + pools []*atev1alpha1.WorkerPool + workers []*ateapipb.Worker + want map[series]int64 + }{ + { + name: "gvisor pool", + pools: []*atev1alpha1.WorkerPool{workerPool("ns-1", "pool-empty", "")}, + workers: []*ateapipb.Worker{ + worker("ns-1", "pool-empty", "", false), + worker("ns-1", "pool-empty", "", false), + worker("ns-1", "pool-empty", "", false), + }, + want: map[series]int64{ + {"ns-1", "pool-empty", ateattr.WorkerStateIdle, gvisor}: 0, + {"ns-1", "pool-empty", ateattr.WorkerStateAssigned, gvisor}: 0, + {"ns-1", "pool-empty", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 3, + }, + }, + { + // No gvisor series shows for a pool that runs no gvisor. + name: "microvm pool", + pools: []*atev1alpha1.WorkerPool{workerPool("ns-1", "pool-micro", atev1alpha1.SandboxClassMicroVM)}, + workers: []*ateapipb.Worker{worker("ns-1", "pool-micro", "", false)}, + want: map[series]int64{ + {"ns-1", "pool-micro", ateattr.WorkerStateIdle, microvm}: 0, + {"ns-1", "pool-micro", ateattr.WorkerStateAssigned, microvm}: 0, + {"ns-1", "pool-micro", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, + }, + }, + { + // A worker that matches no pool has no class to fall back to. + name: "orphan worker", + pools: nil, + workers: []*ateapipb.Worker{worker("ns-1", "pool-orphan", "", false)}, + want: map[series]int64{ + {"ns-1", "pool-orphan", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, + }, + }, } - reader := newWorkerCountReader(t, workers, pools) - sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) - got := seriesCounts(sum) - want := map[series]int64{ - {"ns-1", "pool-orphan", ateattr.WorkerStateIdle, ateattr.SandboxClassUnknown}: 1, - } - if len(got) != len(want) { - t.Fatalf("got %d series, want %d: %v", len(got), len(want), got) - } - for k, v := range want { - if gv, ok := got[k]; !ok || gv != v { - t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pools := func(labels.Selector) ([]*atev1alpha1.WorkerPool, error) { return tt.pools, nil } + workers := func() ([]*ateapipb.Worker, error) { return tt.workers, nil } + reader := newWorkerCountReader(t, workers, pools) + + sum := mustMetric(t, reader, workerpoolWorkersMetric).Data.(metricdata.Sum[int64]) + got := seriesCounts(sum) + if len(got) != len(tt.want) { + t.Fatalf("got %d series, want %d: %v", len(got), len(tt.want), got) + } + for k, v := range tt.want { + if gv, ok := got[k]; !ok || gv != v { + t.Errorf("series %v = %d (present=%v), want %d", k, gv, ok, v) + } + } + }) } } diff --git a/cmd/ateapi/internal/scheduling/metrics.go b/cmd/ateapi/internal/scheduling/metrics.go index 93f1c2a00b..06ee8da2a4 100644 --- a/cmd/ateapi/internal/scheduling/metrics.go +++ b/cmd/ateapi/internal/scheduling/metrics.go @@ -92,7 +92,7 @@ func (s *scheduler) recordEligibleWorkers(ctx context.Context, matching []*ateap s.eligibleWorkers.Record(ctx, count, metric.WithAttributes( ateattr.WorkerPoolNamespaceKey.String(k.namespace), ateattr.WorkerPoolNameKey.String(k.pool), - ateattr.SandboxClassKey.String(constraints.SandboxClass), + ateattr.SandboxClassAttribute(constraints.SandboxClass), ateattr.SchedulingConstraintKey.String(constraintStr), )) } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 9365b21e0f..19dfa4fbdb 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -652,7 +652,7 @@ func TestSchedule_EligibleWorkersMetric(t *testing.T) { } s := New(flt, WithIntn(firstIntn), WithMeter(meter)) - _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "kata"}) + _, err := s.Schedule(context.Background(), Constraints{SandboxClass: "microvm"}) if !errors.Is(err, ErrNoCapacity) { t.Fatalf("Schedule() error = %v, want ErrNoCapacity", err) } @@ -670,8 +670,8 @@ func TestSchedule_EligibleWorkersMetric(t *testing.T) { t.Errorf("datapoint sum = %d, want 0", dp.Sum) } class, _ := dp.Attributes.Value(ateattr.SandboxClassKey) - if class.AsString() != "kata" { - t.Errorf("got sandbox class %q, want kata", class.AsString()) + if class.AsString() != "microvm" { + t.Errorf("got sandbox class %q, want microvm", class.AsString()) } } } diff --git a/internal/ateattr/ateattr.go b/internal/ateattr/ateattr.go index b3abc7dc72..160f70548e 100644 --- a/internal/ateattr/ateattr.go +++ b/internal/ateattr/ateattr.go @@ -343,10 +343,12 @@ func NormalizeSandboxClass(class string) string { } } -// SandboxClassAttribute sets ate.sandbox.class. Use it in place of -// SandboxClassKey: no source of the class is validated, thus all emitters must -// apply the same limit. recordSchedulerAssignment is the one exception. It -// omits the attribute when the class is unknown. +// SandboxClassAttribute sets ate.sandbox.class. No source of the class is +// validated, thus an emitter that sets the attribute must set it through this +// helper and not through SandboxClassKey. +// +// To omit the attribute while the class is unknown is a different choice, and +// two emitters make it: recordSchedulerAssignment and snapshotOp.attrs. func SandboxClassAttribute(class string) attribute.KeyValue { return SandboxClassKey.String(NormalizeSandboxClass(class)) } From 7452f95ebd9353e110ca0d6d792d7d90fcecb204 Mon Sep 17 00:00:00 2001 From: Jeff Luo Date: Wed, 9 Sep 2026 11:06:06 -0400 Subject: [PATCH 4/4] docs: record where ate.sandbox.class reports unknown Four ateapi instruments can now report unknown: the worker count, the crash counter, the lifecycle timers and the eligible-worker histogram. The member's brief described only the atelet source, thus it now names each source. The worker count needs two more lines of its own. A worker with the class unknown is not capacity, thus a query that adds the classes together reports capacity that no resume can use. --- docs/metrics/registry/metrics.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index 7548f2cb1f..45c6317310 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -140,9 +140,11 @@ groups: stability: development value: unknown brief: > - The snapshot manifest has no class, or the class is not in this - list. The default is not gvisor. Thus a bad manifest stays - visible. + The source has no class, or a class that is not in this list. + The default is not gvisor. Thus a bad record stays visible. + ateapi reports it for a Worker that CreateWorker accepted with + no class, and for an ActorTemplate with no sandbox config. + atelet reports it for a snapshot manifest with no class. - id: registry.ate.snapshot type: attribute_group @@ -620,6 +622,9 @@ groups: not write. Some moves into this state are not a loss of data. They are careful responses to a control plane problem. The ate.failure.reason key keeps these groups separate. + ate.sandbox.class is unknown when ateapi could not read the class of the + worker: the worker record is already gone, or its assignment is already + clear. annotations: substrate: emitted_by: [ateapi] @@ -660,6 +665,10 @@ groups: and not a gauge. ateapi sets both states to 0 for each known pool. Thus a full pool or an empty pool reports 0. An absent series would stop an alert on idle == 0. + A worker with the class unknown is not capacity of the pool: the scheduler + puts no actor on it. Thus a query that asks how much capacity a pool has + must keep ate.sandbox.class, because a sum across the classes adds that + worker back into the total. annotations: substrate: emitted_by: [ateapi]