giantswarm · teemow · Sep 11, 2026 · Sep 11, 2026
@@ -95,7 +95,8 @@ internal/lab/                    everything operational:
   oidc.go login.go browser.go      the lab's Dex clients: password grant, authorization-code flow
   test.go                          RBAC assertions for every configured user
   platform.go platformtest.go      agent platform install + the headless MCP proof
-  postrenderers.go                 the lab's per-component postRenderers patches (hostNetwork, sidecar, nodePort, dev images)
+  postrenderers.go                 the lab's per-component postRenderers patches (hostNetwork, sidecar, nodePort, dev-image overrides)
+  devimages.go                     the dev-image swap: image names resolved from the component renders, the lab registry + the digest-pinned Harness image
   fluxreleases.go                  image preload from the chart's rendered OCIRepositories/HelmReleases, each at the version Flux would pull (semverFilter included)
   helm.go restclient.go            the embedded Helm 4 (upgrade-or-install with the kstatus wait, offline renders, probes, uninstalls); the lab kubeconfig as a client-go REST client getter
   resources.go                     docker CPU/memory: the requests table and the floors `up` enforces

@@ -78,36 +78,100 @@ the boot says which chart it installed. The directory is read, never written.
 ## Dev images
 
 To run a component from a build of your own, build the image, and name it
-under `platform.devImages` keyed by the chart's component name (`muster`,
-`backstage`, `kagent` — the controller —, `mcp-kubernetes`, `model-manager`,
-`agent-manager`):
+under `platform.devImages`. The targets are the chart's component names for
+the Deployments the lab can swap — `muster`, `backstage`, `kagent` (the
+controller), `mcp-kubernetes`, `model-manager`, `agent-manager` — and
+`harness`, the platform Harness's runtime image (the Go ADK every agent runs
+on under kagent API v2):
 
 ```yaml
 platform:
   devImages:
     muster: muster:dev-1a2b3c
     backstage: backstage-dev:my-feature-4d5e6f
+    kagent: kagent-controller:dev-7e8f9a      # the line's controller, built from go/ of giantswarm/kagent-upstream
+    harness: golang-adk:dev-7e8f9a            # the Go ADK, same checkout, BUILD_PACKAGE=adk/cmd/main.go
 ```
 
+Either way the swap is part of the release: `agentlab platform` renders it
+into the values it installs, so the same plain `helm upgrade` applies it, and
+removing the entry restores the chart's image on the next run. Use a
+distinctive tag per build — kind's containerd keeps running the old bits
+under a reused tag, and a Harness digest that did not change recompiles
+nothing.
+
+### Deployment targets
+
 `agentlab platform` side-loads the image from the host docker cache into the
 node, checks the node lists it before anything installs (a missing image
 fails right there with its fix, instead of five minutes later as an
 `ImagePullBackOff`, a helm-controller timeout and a rollback to the chart's
 image), and renders it into that component's `postRenderers` as a Kustomize
-image override with `imagePullPolicy: IfNotPresent`, so the swap is part of
-the release: the same plain `helm upgrade` applies it, `kubectl get deploy -o
-jsonpath` shows the new image, and removing the entry restores the chart's
-image on the next run. Use a distinctive tag per build — kind's containerd
-keeps running the old bits under a reused tag. And make it a build of its
-own, not a re-tag of a registry image the kubelet has pulled (a chart that
-pulls `Always`, like Backstage's, makes the kubelet pull): since Kubernetes
-1.33 the kubelet remembers which image IDs it pulled and re-pulls a pod's ref
-that maps to one of them (`KubeletEnsureSecretPulledImages`), so such a
-re-tag ends in `ImagePullBackOff` against Docker Hub while a real build — a
-new image ID, side-loaded, never pulled by the kubelet — is used as is. (A `kubectl patch` on the
-Deployment still works for a quick look, but only until helm-controller's
-next release of that component overwrites it — the values are the durable
-path.)
+image override with `imagePullPolicy: IfNotPresent`; `kubectl get deploy -o
+jsonpath` shows the new image. Make it a build of its own, not a re-tag of a
+registry image the kubelet has pulled (a chart that pulls `Always`, like
+Backstage's, makes the kubelet pull): since Kubernetes 1.33 the kubelet
+remembers which image IDs it pulled and re-pulls a pod's ref that maps to one
+of them (`KubeletEnsureSecretPulledImages`), so such a re-tag ends in
+`ImagePullBackOff` against Docker Hub while a real build — a new image ID,
+side-loaded, never pulled by the kubelet — is used as is. (A `kubectl patch`
+on the Deployment still works for a quick look, but only until
+helm-controller's next release of that component overwrites it — the values
+are the durable path.)
+
+The image name the override replaces is read off the component chart's
+render, the one the boot renders anyway to side-load the platform images —
+not from a table, because it differs between lines: the `kagent` target
+replaces `gsoci.azurecr.io/giantswarm/kagent-controller` under the 3.x meta
+chart (the wrapper chart) and `ghcr.io/giantswarm/kagent/controller` under
+4.x (the kagent line's own chart), whatever Deployment `kagent-controller`'s
+`controller` container names. A Kustomize image override whose name is in no
+rendered object matches nothing, and kustomize drops it without a word — so
+a target whose chart rendered without the Deployment and container the lab
+patches is **refused before the install**, naming both (`platform.devImages.kagent:
+the kagent chart's render has no Deployment kagent-controller with a container
+controller …`); a chart that did not render at all (the preload reports why)
+falls back to the table's name with a note that it is unverified.
+
+### The `harness` target
+
+Under kagent API v2 an agent runs on the platform Harness's workload image,
+not on a Deployment the lab could patch: the connectivity chart renders the
+image **by digest** into the `Harness` object `kagent` (the CRD accepts no
+tag), and Substrate's atelet fetches it from a registry into its own layer
+cache — the actors' overlay lowerdirs — never through the node's containerd,
+so a side-load is invisible to it. The lab therefore runs a **registry**
+for this target: a `registry` container named `<clusterName>-registry` on
+the kind docker network, published on the host's loopback
+(`platform.devRegistryPort`, default 5001, kind's documented local-registry
+port; created on demand, removed by `agentlab down`). `agentlab platform`
+tags the build into it (`localhost:<port>/<path of your ref>:<tag>`), pushes,
+reads the manifest digest the registry computed, and forwards
+`kagent.harness.image: localhost:<port>/<path>@sha256:<digest>` through the
+meta chart to the connectivity chart. atelet's side of the pattern,
+`--localhost-registry-replacement=<clusterName>-registry:5000` (a `localhost`
+registry in an image ref is rewritten to that endpoint and pulled over plain
+HTTP), is part of the lab shape whenever the agents are on — through the
+meta chart's `substrate.atelet.extraArgs`, forwarded to its substrate
+component — not only while a dev image is configured: the flag is inert for every other
+ref, and having it in place before a swap means the Harness's new digest
+(the connectivity release) can never race ahead of the atelet roll (the
+substrate release) that would carry it. The boot then asserts the Harness
+pins the dev digest, and — because a **digest-pinned Harness recompiles
+every template it admits** (a new golden snapshot per revision) — lists the
+admitted templates as they come back Ready on it
+(`Harness kagent runs localhost:5001/golang-adk@sha256:…; 1 admitted templates
+recompiled on it: my-agent 88c11e… -> ebfce6…`). Removing the entry restores
+the chart's digest on the next run — again one `helm upgrade`, again a
+recompile. A `platform.valuesFiles` overlay that sets `kagent.harness.image`
+itself (the way a lab pins a published build) or replaces
+`substrate.atelet.extraArgs` (lists replace in a Helm merge) is refused while
+the target is configured.
+
+Because atelet keys its cache by digest, the kubelet re-pull trap above does
+not apply here, and a pushed image is pulled once per node. The mechanism
+needs the platform Harness, so the target is for the 4.x meta chart with
+the agents on; the 3.x line has no Harness.
 
 muster runs with `hostNetwork`, so it binds `:8090` on the node, and the
 rendered kind config publishes that onto the Mac (host port `platform.musterPort`,

@@ -81,6 +81,30 @@ func TestChartSourceValidation(t *testing.T) {
 	if err := cfg.Validate(); err != nil {
 		t.Errorf("devImages for muster, backstage, kagent: %v", err)
 	}
+	// The harness target is the platform Harness's image: it comes with the
+	// agents, and the registry that serves it has a host port of its own.
+	cfg.Platform.DevImages = map[string]string{DevImageHarness: "golang-adk:dev-139"}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("devImages.harness with agents on: %v", err)
+	}
+	if cfg.Platform.DevRegistryPort != DefaultDevRegistryPort {
+		t.Errorf("default devRegistryPort = %d, want %d", cfg.Platform.DevRegistryPort, DefaultDevRegistryPort)
+	}
+	cfg.Platform.DevRegistryPort = 0
+	cfg.Normalize()
+	if cfg.Platform.DevRegistryPort != DefaultDevRegistryPort {
+		t.Errorf("Normalize left devRegistryPort %d, want the default", cfg.Platform.DevRegistryPort)
+	}
+	cfg.Platform.DevRegistryPort = 70000
+	if err := cfg.Validate(); err == nil {
+		t.Error("devRegistryPort out of range: want an error")
+	}
+	cfg.Platform.DevRegistryPort = DefaultDevRegistryPort
+	cfg.Platform.Agents = false
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "platform.agents") {
+		t.Errorf("devImages.harness without agents: want the error naming platform.agents, got %v", err)
+	}
+	cfg.Platform.Agents = true
 
 	cfg.Platform.ValuesFiles = []string{filepath.Join(t.TempDir(), "missing.yaml")}
 	if err := cfg.Validate(); err == nil {

@@ -99,10 +99,23 @@ const DefaultChartVersion = "4.7.11"
 // ChartRepository is where the agent-platform chart releases live.
 const ChartRepository = "oci://gsoci.azurecr.io/charts/giantswarm/agent-platform"
 
-// DevImageComponents are the components whose image platform.devImages can
-// swap: the agent-platform chart's component names (its `components.<name>`
-// entries) for the workloads the lab's dev loops build from a checkout.
-var DevImageComponents = []string{"muster", "backstage", "kagent", "mcp-kubernetes", "model-manager", "agent-manager"}
+// DevImageComponents are the targets platform.devImages can swap: the
+// agent-platform chart's component names (its `components.<name>` entries)
+// for the Deployments the lab's dev loops build from a checkout, plus
+// DevImageHarness for the platform Harness's runtime image.
+var DevImageComponents = []string{"muster", "backstage", "kagent", "mcp-kubernetes", "model-manager", "agent-manager", DevImageHarness}
+
+// DevImageHarness is the devImages key of the platform Harness's workload
+// image — the Go ADK runtime every agent runs on under kagent API v2. Not a
+// Deployment: the connectivity chart renders the image by digest into the
+// `Harness` object, and Substrate's atelet pulls it from a registry into its
+// own layer cache, so the lab pushes the local build to its registry
+// (DevRegistryPort) and forwards the digest through kagent.harness.image.
+const DevImageHarness = "harness"
+
+// DefaultDevRegistryPort is the host port of the lab registry when
+// agentlab.yaml sets none: kind's documented local-registry port.
+const DefaultDevRegistryPort = 5001
 
 type User struct {
 	Email        string   `yaml:"email"`
@@ -187,14 +200,27 @@ type Platform struct {
 	// (or the key is dropped). Meaningless without chartBranch.
 	ChartPinned bool `yaml:"chartPinned,omitempty"`
 	// DevImages swaps a component's image for a build of your own (the lab's
-	// dev-image loop): component name -> image ref (`muster: muster:dev-1a2b`).
-	// `agentlab platform` side-loads the ref from the host docker cache and
-	// renders it into the component's HelmRelease as a kustomize image
-	// override (postRenderers) with imagePullPolicy IfNotPresent, so the swap
-	// is part of the release — a plain `helm upgrade` applies it, and removing
-	// the entry restores the chart's image on the next run. Keys are the
-	// DevImageComponents.
+	// dev-image loop): target -> image ref (`muster: muster:dev-1a2b`). Keys
+	// are the DevImageComponents. For a Deployment target `agentlab platform`
+	// side-loads the ref from the host docker cache, resolves the image name
+	// it replaces from the component chart's render (the kagent controller is
+	// `ghcr.io/giantswarm/kagent/controller` on the 4.x line and
+	// `gsoci.azurecr.io/giantswarm/kagent-controller` on 3.x — a target that
+	// matches nothing in the render is an error before the install, never a
+	// silently dropped override) and renders it into the component's
+	// HelmRelease as a kustomize image override (postRenderers) with
+	// imagePullPolicy IfNotPresent. For the `harness` target it pushes the
+	// build to the lab registry and forwards the digest as
+	// kagent.harness.image, so the platform Harness runs it and recompiles
+	// every admitted template. Either way the swap is part of the release — a
+	// plain `helm upgrade` applies it, and removing the entry restores the
+	// chart's image (or digest) on the next run.
 	DevImages map[string]string `yaml:"devImages,omitempty"`
+	// DevRegistryPort is the host port (127.0.0.1) of the lab registry that
+	// serves the `harness` dev image: a `registry` container on the kind
+	// docker network, created on demand by `agentlab platform` and removed by
+	// `agentlab down`. Unset means DefaultDevRegistryPort.
+	DevRegistryPort int `yaml:"devRegistryPort,omitempty"`
 	// ValuesFiles are extra Helm values files merged over the lab's rendered
 	// values before the meta chart install, in order, with `helm -f`
 	// semantics (maps merge, lists replace, the later file wins): a lab that
@@ -466,6 +492,9 @@ func Default() *Config {
 			Domain:        "127.0.0.1.nip.io",
 			GatewayPort:   443,
 			ChartVersion:  DefaultChartVersion,
+			// The lab registry behind the `harness` dev image; a container
+			// on the kind network, so no node port mapping is involved.
+			DevRegistryPort: DefaultDevRegistryPort,
 		},
 		Backstage: Backstage{
 			Enabled: true,
@@ -640,6 +669,9 @@ func (c *Config) Normalize() {
 	if len(c.Platform.DevImages) == 0 {
 		c.Platform.DevImages = nil
 	}
+	if c.Platform.DevRegistryPort == 0 {
+		c.Platform.DevRegistryPort = DefaultDevRegistryPort
+	}
 	if len(c.Platform.ValuesFiles) == 0 {
 		c.Platform.ValuesFiles = nil
 	}
@@ -804,12 +836,20 @@ func (c *Config) Validate() error {
 	}
 	for component, ref := range c.Platform.DevImages {
 		if !slices.Contains(DevImageComponents, component) {
-			return fmt.Errorf("platform.devImages: unknown component %q (one of %s)", component, strings.Join(DevImageComponents, ", "))
+			return fmt.Errorf("platform.devImages: unknown target %q (one of %s)", component, strings.Join(DevImageComponents, ", "))
 		}
 		if err := ValidateImageRef(ref); err != nil {
 			return fmt.Errorf("platform.devImages.%s %q: %w", component, ref, err)
 		}
 	}
+	if _, ok := c.Platform.DevImages[DevImageHarness]; ok && !c.Platform.Agents {
+		return fmt.Errorf("platform.devImages.%s: the platform Harness comes with the agents (platform.agents: true)", DevImageHarness)
+	}
+	if c.Platform.DevRegistryPort != 0 {
+		if err := ValidatePort(strconv.Itoa(c.Platform.DevRegistryPort)); err != nil {
+			return fmt.Errorf("platform.devRegistryPort: %w", err)
+		}
+	}
 	for _, path := range c.Platform.ValuesFiles {
 		if path == "" {
 			return fmt.Errorf("platform.valuesFiles: an empty path")

@@ -52,7 +52,7 @@ const (
 // dexSANNames are the DNS names of the Dex server cert: the issuer name
 // (localhost — see docs/identity.md on why not 127.0.0.1) plus the in-cluster
 // service names. Also the fixed half of the CA's name constraints.
-var dexSANNames = []string{"localhost", componentDex, "dex.dex.svc", "dex.dex.svc.cluster.local"}
+var dexSANNames = []string{localhostName, componentDex, "dex.dex.svc", "dex.dex.svc.cluster.local"}
 
 // permittedDNSNames is the CA's DNS name-constraint allowlist: the platform
 // domain plus the fixed lab names. Everything a lab leaf will ever carry,