From 41790f72ddfa2309d6647abb0619c21d556cabed Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:33:42 +0000
Subject: [PATCH 1/5] Serialize OCI cache index updates

layout.AppendImage does an unsynchronized read-modify-write of index.json.
Overlapping appends from registry pushes, remote pulls, and builder imports
lost descriptors or left the index malformed. Route all three through
ocicache.AppendImage, which writes blobs concurrently and holds a mutex
around the index append.
---
 lib/builds/manager.go       | 15 ++-------
 lib/images/oci.go           | 20 ++----------
 lib/ocicache/layout.go      | 44 ++++++++++++++++++++++++++
 lib/ocicache/layout_test.go | 62 +++++++++++++++++++++++++++++++++++++
 lib/registry/registry.go    | 14 +--------
 5 files changed, 111 insertions(+), 44 deletions(-)
 create mode 100644 lib/ocicache/layout.go
 create mode 100644 lib/ocicache/layout_test.go

diff --git a/lib/builds/manager.go b/lib/builds/manager.go
index b9157d93c..806ae5622 100644
--- a/lib/builds/manager.go
+++ b/lib/builds/manager.go
@@ -16,12 +16,11 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/kernel/hypeman/lib/builders"
 	"github.com/kernel/hypeman/lib/images"
 	"github.com/kernel/hypeman/lib/instances"
+	"github.com/kernel/hypeman/lib/ocicache"
 	"github.com/kernel/hypeman/lib/paths"
 	"github.com/kernel/hypeman/lib/tags"
 	"github.com/kernel/hypeman/lib/volumes"
@@ -382,17 +381,7 @@ func (m *manager) buildBuilderFromDockerfile(ctx context.Context) (string, error
 	// ImportLocalImage triggers buildImage → pullAndExport, it will find the
 	// layers already cached and skip the network pull entirely.
 	cacheDir := m.paths.SystemOCICache()
-	layoutPath, err := layout.FromPath(cacheDir)
-	if err != nil {
-		layoutPath, err = layout.Write(cacheDir, empty.Index)
-		if err != nil {
-			return "", fmt.Errorf("create OCI layout: %w", err)
-		}
-	}
-
-	if err := layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{
-		"org.opencontainers.image.ref.name": digestHex,
-	})); err != nil {
+	if err := ocicache.AppendImage(cacheDir, img, digestHex); err != nil {
 		return "", fmt.Errorf("add image to OCI layout: %w", err)
 	}
 
diff --git a/lib/images/oci.go b/lib/images/oci.go
index 36684cacd..c230db19f 100644
--- a/lib/images/oci.go
+++ b/lib/images/oci.go
@@ -10,9 +10,9 @@ import (
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
 	gcr "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/kernel/hypeman/lib/ocicache"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -318,23 +318,7 @@ func (c *ociClient) pullToOCILayoutWithPlatformAuth(ctx context.Context, imageRe
 		return fmt.Errorf("fetch image manifest: %w", ClassifyRegistryError(err))
 	}
 
-	// Open or create OCI layout directory
-	path, err := layout.FromPath(c.cacheDir)
-	if err != nil {
-		// If layout doesn't exist, create it
-		path, err = layout.Write(c.cacheDir, empty.Index)
-		if err != nil {
-			return fmt.Errorf("create oci layout: %w", err)
-		}
-	}
-
-	// Append image to layout - THIS is where actual layer data is downloaded
-	// Streams layers from registry and writes to blobs/sha256/ directory
-	// Automatically deduplicates shared layers across images
-	// Rate limits during layer download also fail immediately (no retries)
-	err = path.AppendImage(img, layout.WithAnnotations(map[string]string{
-		"org.opencontainers.image.ref.name": layoutTag,
-	}))
+	err = ocicache.AppendImage(c.cacheDir, img, layoutTag)
 	if err != nil {
 		return fmt.Errorf("download and write image layers: %w", err)
 	}
diff --git a/lib/ocicache/layout.go b/lib/ocicache/layout.go
new file mode 100644
index 000000000..b6ee37732
--- /dev/null
+++ b/lib/ocicache/layout.go
@@ -0,0 +1,44 @@
+package ocicache
+
+import (
+	"fmt"
+	"sync"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
+)
+
+// indexMu serializes index.json updates. layout.AppendDescriptor does an
+// unsynchronized read-modify-write of index.json, so concurrent appends lose
+// descriptors or corrupt the file.
+var indexMu sync.Mutex
+
+// AppendImage writes img's blobs to the OCI layout at cacheDir, then appends a
+// descriptor tagged with org.opencontainers.image.ref.name=tag to index.json.
+// Blob writes run concurrently; only the index update is serialized.
+func AppendImage(cacheDir string, img v1.Image, tag string) error {
+	path := layout.Path(cacheDir)
+	if err := path.WriteImage(img); err != nil {
+		return fmt.Errorf("write image blobs: %w", err)
+	}
+	desc, err := partial.Descriptor(img)
+	if err != nil {
+		return fmt.Errorf("describe image: %w", err)
+	}
+	if desc.Annotations == nil {
+		desc.Annotations = make(map[string]string)
+	}
+	desc.Annotations["org.opencontainers.image.ref.name"] = tag
+
+	indexMu.Lock()
+	defer indexMu.Unlock()
+
+	if _, err := layout.FromPath(cacheDir); err != nil {
+		if _, err := layout.Write(cacheDir, empty.Index); err != nil {
+			return fmt.Errorf("create OCI layout: %w", err)
+		}
+	}
+	return path.AppendDescriptor(*desc)
+}
diff --git a/lib/ocicache/layout_test.go b/lib/ocicache/layout_test.go
new file mode 100644
index 000000000..8aaf2f9ee
--- /dev/null
+++ b/lib/ocicache/layout_test.go
@@ -0,0 +1,62 @@
+package ocicache
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/google/go-containerregistry/pkg/v1/random"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppendImageConcurrent(t *testing.T) {
+	cacheDir := t.TempDir()
+	const count = 64
+	images := make([]v1.Image, count)
+	for i := range images {
+		var err error
+		images[i], err = random.Image(128, 1)
+		require.NoError(t, err)
+	}
+
+	start := make(chan struct{})
+	errs := make(chan error, count)
+	var writers sync.WaitGroup
+	for i, img := range images {
+		writers.Add(1)
+		go func() {
+			defer writers.Done()
+			<-start
+			errs <- AppendImage(cacheDir, img, fmt.Sprintf("image-%d", i))
+		}()
+	}
+	close(start)
+	writers.Wait()
+	close(errs)
+	for err := range errs {
+		require.NoError(t, err)
+	}
+
+	path, err := layout.FromPath(cacheDir)
+	require.NoError(t, err)
+	index, err := path.ImageIndex()
+	require.NoError(t, err)
+	manifest, err := index.IndexManifest()
+	require.NoError(t, err)
+	require.Len(t, manifest.Manifests, count)
+	tags := make(map[string]v1.Hash, count)
+	for _, desc := range manifest.Manifests {
+		tags[desc.Annotations["org.opencontainers.image.ref.name"]] = desc.Digest
+	}
+	for i, img := range images {
+		digest, err := img.Digest()
+		require.NoError(t, err)
+		require.Equal(t, digest, tags[fmt.Sprintf("image-%d", i)])
+		cached, err := path.Image(digest)
+		require.NoError(t, err)
+		_, err = cached.ConfigFile()
+		require.NoError(t, err)
+	}
+}
diff --git a/lib/registry/registry.go b/lib/registry/registry.go
index 5ac66310f..2728e44a9 100644
--- a/lib/registry/registry.go
+++ b/lib/registry/registry.go
@@ -16,8 +16,6 @@ import (
 	"sync"
 
 	"github.com/google/go-containerregistry/pkg/registry"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/kernel/hypeman/lib/images"
 	"github.com/kernel/hypeman/lib/ocicache"
 	"github.com/kernel/hypeman/lib/paths"
@@ -218,14 +216,6 @@ func (r *Registry) triggerConversion(repo, reference, dockerDigest string) {
 // addToOCILayout adds the image to the OCI layout, converting Docker v2 to OCI if needed.
 func (r *Registry) addToOCILayout(inputDigest string) (string, error) {
 	cacheDir := r.paths.SystemOCICache()
-	path, err := layout.FromPath(cacheDir)
-	if err != nil {
-		path, err = layout.Write(cacheDir, empty.Index)
-		if err != nil {
-			return "", fmt.Errorf("create oci layout: %w", err)
-		}
-	}
-
 	img, err := ocicache.ImageFromCache(r.paths, inputDigest)
 	if err != nil {
 		return "", fmt.Errorf("create image from blob store: %w", err)
@@ -238,9 +228,7 @@ func (r *Registry) addToOCILayout(inputDigest string) (string, error) {
 	digest := digestHash.String()
 	digestHex := digestHash.Hex
 
-	err = path.AppendImage(img, layout.WithAnnotations(map[string]string{
-		"org.opencontainers.image.ref.name": digestHex,
-	}))
+	err = ocicache.AppendImage(cacheDir, img, digestHex)
 	if err != nil {
 		return "", fmt.Errorf("append image to layout: %w", err)
 	}

From 4da048d58d06bc36243da4c60b0d3027fcffffff Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Fri, 9 Oct 2026 16:33:51 +0000
Subject: [PATCH 2/5] Only initialize OCI layout when index.json is missing

Any other FromPath error used to fall through to layout.Write, which
replaces index.json with an empty index. Also copy the descriptor before
annotating so remote images keep their own descriptor untouched.
---
 lib/ocicache/layout.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/ocicache/layout.go b/lib/ocicache/layout.go
index b6ee37732..e8a83a110 100644
--- a/lib/ocicache/layout.go
+++ b/lib/ocicache/layout.go
@@ -1,7 +1,9 @@
 package ocicache
 
 import (
+	"errors"
 	"fmt"
+	"io/fs"
 	"sync"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -23,12 +25,15 @@ func AppendImage(cacheDir string, img v1.Image, tag string) error {
 	if err := path.WriteImage(img); err != nil {
 		return fmt.Errorf("write image blobs: %w", err)
 	}
-	desc, err := partial.Descriptor(img)
+	retained, err := partial.Descriptor(img)
 	if err != nil {
 		return fmt.Errorf("describe image: %w", err)
 	}
-	if desc.Annotations == nil {
-		desc.Annotations = make(map[string]string)
+	// Copy before annotating: remote images hand back their own descriptor.
+	desc := *retained
+	desc.Annotations = make(map[string]string, len(retained.Annotations)+1)
+	for k, v := range retained.Annotations {
+		desc.Annotations[k] = v
 	}
 	desc.Annotations["org.opencontainers.image.ref.name"] = tag
 
@@ -36,9 +41,12 @@ func AppendImage(cacheDir string, img v1.Image, tag string) error {
 	defer indexMu.Unlock()
 
 	if _, err := layout.FromPath(cacheDir); err != nil {
+		if !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("open OCI layout: %w", err)
+		}
 		if _, err := layout.Write(cacheDir, empty.Index); err != nil {
 			return fmt.Errorf("create OCI layout: %w", err)
 		}
 	}
-	return path.AppendDescriptor(*desc)
+	return path.AppendDescriptor(desc)
 }

From 5767a6ef6a1b54de81da3a493b704fb02db1d124 Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Fri, 9 Oct 2026 17:39:16 +0000
Subject: [PATCH 3/5] Publish OCI cache blobs and index atomically

Write every cache file to a temp file and rename it into place. index.json
no longer ends up empty or truncated after a crash or ENOSPC mid-append,
and unlocked readers never see a partial index. Config and manifest blobs
previously went straight to their final path and were skipped on any
existing file; a blob truncated by a killed writer was trusted forever.
Blobs are now skipped only when the existing size matches.
---
 lib/ocicache/layout.go      | 166 ++++++++++++++++++++++++++++++++----
 lib/ocicache/layout_test.go | 146 +++++++++++++++++++++++++++++--
 2 files changed, 289 insertions(+), 23 deletions(-)

diff --git a/lib/ocicache/layout.go b/lib/ocicache/layout.go
index e8a83a110..113da1f6e 100644
--- a/lib/ocicache/layout.go
+++ b/lib/ocicache/layout.go
@@ -1,29 +1,35 @@
 package ocicache
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
+	"os"
+	"path/filepath"
 	"sync"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
+	"golang.org/x/sync/errgroup"
 )
 
-// indexMu serializes index.json updates. layout.AppendDescriptor does an
-// unsynchronized read-modify-write of index.json, so concurrent appends lose
-// descriptors or corrupt the file.
+// indexMu serializes index.json read-modify-write cycles. Without it,
+// concurrent appends lose descriptors or corrupt the file.
 var indexMu sync.Mutex
 
-// AppendImage writes img's blobs to the OCI layout at cacheDir, then appends a
-// descriptor tagged with org.opencontainers.image.ref.name=tag to index.json.
-// Blob writes run concurrently; only the index update is serialized.
+// AppendImage writes img's blobs into the OCI layout at cacheDir, then appends
+// a descriptor tagged with org.opencontainers.image.ref.name=tag to index.json.
+//
+// Every file is written to a temp file and renamed into place, so readers never
+// see a partial blob or index and an interrupted write leaves nothing behind.
+// Blob writes run concurrently; the index update is serialized.
 func AppendImage(cacheDir string, img v1.Image, tag string) error {
-	path := layout.Path(cacheDir)
-	if err := path.WriteImage(img); err != nil {
-		return fmt.Errorf("write image blobs: %w", err)
+	if err := writeImage(cacheDir, img); err != nil {
+		return err
 	}
 	retained, err := partial.Descriptor(img)
 	if err != nil {
@@ -39,14 +45,144 @@ func AppendImage(cacheDir string, img v1.Image, tag string) error {
 
 	indexMu.Lock()
 	defer indexMu.Unlock()
+	return appendDescriptor(cacheDir, desc)
+}
+
+func writeImage(cacheDir string, img v1.Image) error {
+	layers, err := img.Layers()
+	if err != nil {
+		return fmt.Errorf("list layers: %w", err)
+	}
+	var g errgroup.Group
+	for _, layer := range layers {
+		g.Go(func() error {
+			digest, err := layer.Digest()
+			if err != nil {
+				return err
+			}
+			size, err := layer.Size()
+			if err != nil {
+				return err
+			}
+			rc, err := layer.Compressed()
+			if err != nil {
+				return err
+			}
+			return writeBlob(cacheDir, digest, size, rc)
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return fmt.Errorf("write layers: %w", err)
+	}
+
+	cfgName, err := img.ConfigName()
+	if err != nil {
+		return err
+	}
+	cfg, err := img.RawConfigFile()
+	if err != nil {
+		return err
+	}
+	if err := writeBlob(cacheDir, cfgName, int64(len(cfg)), io.NopCloser(bytes.NewReader(cfg))); err != nil {
+		return fmt.Errorf("write config: %w", err)
+	}
+
+	digest, err := img.Digest()
+	if err != nil {
+		return err
+	}
+	manifest, err := img.RawManifest()
+	if err != nil {
+		return err
+	}
+	if err := writeBlob(cacheDir, digest, int64(len(manifest)), io.NopCloser(bytes.NewReader(manifest))); err != nil {
+		return fmt.Errorf("write manifest: %w", err)
+	}
+	return nil
+}
 
-	if _, err := layout.FromPath(cacheDir); err != nil {
-		if !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("open OCI layout: %w", err)
+// writeBlob skips the write when the blob already exists with the expected
+// size, so a blob truncated by an interrupted writer is rewritten.
+func writeBlob(cacheDir string, hash v1.Hash, size int64, rc io.ReadCloser) error {
+	defer rc.Close()
+	dir := filepath.Join(cacheDir, "blobs", hash.Algorithm)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	final := filepath.Join(dir, hash.Hex)
+	if info, err := os.Stat(final); err == nil && info.Size() == size {
+		return nil
+	}
+	return writeFile(final, func(w io.Writer) error {
+		n, err := io.Copy(w, rc)
+		if err != nil {
+			return err
+		}
+		if n != size {
+			return fmt.Errorf("blob %s: wrote %d bytes, expected %d", hash, n, size)
 		}
-		if _, err := layout.Write(cacheDir, empty.Index); err != nil {
+		return nil
+	})
+}
+
+func appendDescriptor(cacheDir string, desc v1.Descriptor) error {
+	layoutPath := filepath.Join(cacheDir, "oci-layout")
+	if _, err := os.Stat(layoutPath); errors.Is(err, fs.ErrNotExist) {
+		err = writeFile(layoutPath, func(w io.Writer) error {
+			_, err := io.WriteString(w, `{"imageLayoutVersion":"1.0.0"}`)
+			return err
+		})
+		if err != nil {
 			return fmt.Errorf("create OCI layout: %w", err)
 		}
+	} else if err != nil {
+		return fmt.Errorf("open OCI layout: %w", err)
+	}
+
+	indexPath := filepath.Join(cacheDir, "index.json")
+	index, err := empty.Index.IndexManifest()
+	if err != nil {
+		return err
+	}
+	data, err := os.ReadFile(indexPath)
+	if err == nil {
+		err = json.Unmarshal(data, index)
+	} else if errors.Is(err, fs.ErrNotExist) {
+		err = nil
+	}
+	if err != nil {
+		return fmt.Errorf("read OCI cache index: %w", err)
+	}
+	index.Manifests = append(index.Manifests, desc)
+	data, err = json.MarshalIndent(index, "", "   ")
+	if err != nil {
+		return fmt.Errorf("encode OCI cache index: %w", err)
+	}
+	return writeFile(indexPath, func(w io.Writer) error {
+		_, err := w.Write(data)
+		return err
+	})
+}
+
+// writeFile writes to a temp file in the same directory and renames it over
+// path. The temp name never matches a blob digest, so the cache GC ignores it.
+func writeFile(path string, write func(io.Writer) error) error {
+	dir, base := filepath.Split(path)
+	tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	if err := tmp.Chmod(0o644); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := write(tmp); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
 	}
-	return path.AppendDescriptor(desc)
+	return os.Rename(tmp.Name(), path)
 }
diff --git a/lib/ocicache/layout_test.go b/lib/ocicache/layout_test.go
index 8aaf2f9ee..86bd3a565 100644
--- a/lib/ocicache/layout_test.go
+++ b/lib/ocicache/layout_test.go
@@ -1,7 +1,10 @@
 package ocicache
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 
@@ -39,12 +42,7 @@ func TestAppendImageConcurrent(t *testing.T) {
 		require.NoError(t, err)
 	}
 
-	path, err := layout.FromPath(cacheDir)
-	require.NoError(t, err)
-	index, err := path.ImageIndex()
-	require.NoError(t, err)
-	manifest, err := index.IndexManifest()
-	require.NoError(t, err)
+	manifest := readIndex(t, cacheDir)
 	require.Len(t, manifest.Manifests, count)
 	tags := make(map[string]v1.Hash, count)
 	for _, desc := range manifest.Manifests {
@@ -54,9 +52,141 @@ func TestAppendImageConcurrent(t *testing.T) {
 		digest, err := img.Digest()
 		require.NoError(t, err)
 		require.Equal(t, digest, tags[fmt.Sprintf("image-%d", i)])
-		cached, err := path.Image(digest)
+		requireImageReadable(t, cacheDir, digest)
+	}
+	requireNoTempFiles(t, cacheDir)
+}
+
+func TestAppendImageSameImageConcurrent(t *testing.T) {
+	cacheDir := t.TempDir()
+	img, err := random.Image(128, 2)
+	require.NoError(t, err)
+	digest, err := img.Digest()
+	require.NoError(t, err)
+
+	const count = 16
+	start := make(chan struct{})
+	errs := make(chan error, count)
+	var writers sync.WaitGroup
+	for i := range count {
+		writers.Add(1)
+		go func() {
+			defer writers.Done()
+			<-start
+			errs <- AppendImage(cacheDir, img, fmt.Sprintf("tag-%d", i))
+		}()
+	}
+	close(start)
+	writers.Wait()
+	close(errs)
+	for err := range errs {
+		require.NoError(t, err)
+	}
+
+	manifest := readIndex(t, cacheDir)
+	require.Len(t, manifest.Manifests, count)
+	for _, desc := range manifest.Manifests {
+		require.Equal(t, digest, desc.Digest)
+	}
+	requireImageReadable(t, cacheDir, digest)
+	requireNoTempFiles(t, cacheDir)
+}
+
+func TestAppendImageRewritesTruncatedBlob(t *testing.T) {
+	cacheDir := t.TempDir()
+	img, err := random.Image(128, 1)
+	require.NoError(t, err)
+	require.NoError(t, AppendImage(cacheDir, img, "first"))
+
+	cfgName, err := img.ConfigName()
+	require.NoError(t, err)
+	cfgPath := filepath.Join(cacheDir, "blobs", cfgName.Algorithm, cfgName.Hex)
+	require.NoError(t, os.WriteFile(cfgPath, []byte("{"), 0o644))
+
+	require.NoError(t, AppendImage(cacheDir, img, "second"))
+	cfg, err := img.RawConfigFile()
+	require.NoError(t, err)
+	got, err := os.ReadFile(cfgPath)
+	require.NoError(t, err)
+	require.Equal(t, cfg, got)
+}
+
+func TestAppendImageAtomicIndexReaders(t *testing.T) {
+	cacheDir := t.TempDir()
+	img, err := random.Image(128, 1)
+	require.NoError(t, err)
+	for i := range 200 {
+		require.NoError(t, AppendImage(cacheDir, img, fmt.Sprintf("seed-%d", i)))
+	}
+
+	done := make(chan struct{})
+	readerErr := make(chan error, 1)
+	go func() {
+		indexPath := filepath.Join(cacheDir, "index.json")
+		for {
+			select {
+			case <-done:
+				readerErr <- nil
+				return
+			default:
+			}
+			data, err := os.ReadFile(indexPath)
+			if err == nil && !json.Valid(data) {
+				err = fmt.Errorf("reader observed an invalid index")
+			}
+			if err != nil {
+				readerErr <- err
+				return
+			}
+		}
+	}()
+	for i := range 64 {
+		if err := AppendImage(cacheDir, img, fmt.Sprintf("tag-%d", i)); err != nil {
+			t.Error(err)
+			break
+		}
+	}
+	close(done)
+	require.NoError(t, <-readerErr)
+}
+
+func readIndex(t *testing.T, cacheDir string) *v1.IndexManifest {
+	t.Helper()
+	path, err := layout.FromPath(cacheDir)
+	require.NoError(t, err)
+	index, err := path.ImageIndex()
+	require.NoError(t, err)
+	manifest, err := index.IndexManifest()
+	require.NoError(t, err)
+	return manifest
+}
+
+func requireImageReadable(t *testing.T, cacheDir string, digest v1.Hash) {
+	t.Helper()
+	path, err := layout.FromPath(cacheDir)
+	require.NoError(t, err)
+	img, err := path.Image(digest)
+	require.NoError(t, err)
+	_, err = img.ConfigFile()
+	require.NoError(t, err)
+	layers, err := img.Layers()
+	require.NoError(t, err)
+	for _, layer := range layers {
+		size, err := layer.Size()
+		require.NoError(t, err)
+		digest, err := layer.Digest()
 		require.NoError(t, err)
-		_, err = cached.ConfigFile()
+		info, err := os.Stat(filepath.Join(cacheDir, "blobs", digest.Algorithm, digest.Hex))
+		require.NoError(t, err)
+		require.Equal(t, size, info.Size())
+	}
+}
+
+func requireNoTempFiles(t *testing.T, cacheDir string) {
+	t.Helper()
+	for _, pattern := range []string{".*.tmp-*", "blobs/sha256/.*.tmp-*"} {
+		files, err := filepath.Glob(filepath.Join(cacheDir, pattern))
 		require.NoError(t, err)
+		require.Empty(t, files)
 	}
 }

From 0c19f674b4c7e2a45ed3c6a6d8e8c417a5d98071 Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:11:21 +0000
Subject: [PATCH 4/5] Sync cache files before rename and store registry
 manifests atomically

fsync the temp file so a renamed index or blob survives power loss on
filesystems without flush-on-rename. Route the registry's Docker manifest
blob through the same writer instead of os.WriteFile on the final path.
---
 lib/ocicache/layout.go   | 19 +++++++++++++++----
 lib/registry/registry.go |  4 ++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/lib/ocicache/layout.go b/lib/ocicache/layout.go
index 113da1f6e..8b5e4b070 100644
--- a/lib/ocicache/layout.go
+++ b/lib/ocicache/layout.go
@@ -25,8 +25,8 @@ var indexMu sync.Mutex
 // a descriptor tagged with org.opencontainers.image.ref.name=tag to index.json.
 //
 // Every file is written to a temp file and renamed into place, so readers never
-// see a partial blob or index and an interrupted write leaves nothing behind.
-// Blob writes run concurrently; the index update is serialized.
+// see a partial blob or index and an interrupted write never replaces a good
+// file. Blob writes run concurrently; the index update is serialized.
 func AppendImage(cacheDir string, img v1.Image, tag string) error {
 	if err := writeImage(cacheDir, img); err != nil {
 		return err
@@ -101,6 +101,12 @@ func writeImage(cacheDir string, img v1.Image) error {
 	return nil
 }
 
+// WriteBlob stores data as the blob for hash, which the caller must have
+// verified. Atomic and concurrency-safe like the blobs AppendImage writes.
+func WriteBlob(cacheDir string, hash v1.Hash, data []byte) error {
+	return writeBlob(cacheDir, hash, int64(len(data)), io.NopCloser(bytes.NewReader(data)))
+}
+
 // writeBlob skips the write when the blob already exists with the expected
 // size, so a blob truncated by an interrupted writer is rewritten.
 func writeBlob(cacheDir string, hash v1.Hash, size int64, rc io.ReadCloser) error {
@@ -164,8 +170,9 @@ func appendDescriptor(cacheDir string, desc v1.Descriptor) error {
 	})
 }
 
-// writeFile writes to a temp file in the same directory and renames it over
-// path. The temp name never matches a blob digest, so the cache GC ignores it.
+// writeFile writes to a temp file in the same directory, syncs it, and renames
+// it over path. The temp name never matches a blob digest, so the cache GC
+// ignores it.
 func writeFile(path string, write func(io.Writer) error) error {
 	dir, base := filepath.Split(path)
 	tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
@@ -181,6 +188,10 @@ func writeFile(path string, write func(io.Writer) error) error {
 		tmp.Close()
 		return err
 	}
+	if err := tmp.Sync(); err != nil {
+		tmp.Close()
+		return err
+	}
 	if err := tmp.Close(); err != nil {
 		return err
 	}
diff --git a/lib/registry/registry.go b/lib/registry/registry.go
index 2728e44a9..77249b920 100644
--- a/lib/registry/registry.go
+++ b/lib/registry/registry.go
@@ -16,6 +16,7 @@ import (
 	"sync"
 
 	"github.com/google/go-containerregistry/pkg/registry"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/kernel/hypeman/lib/images"
 	"github.com/kernel/hypeman/lib/ocicache"
 	"github.com/kernel/hypeman/lib/paths"
@@ -163,7 +164,6 @@ func (r *Registry) Handler() http.Handler {
 // storeManifestBlob stores a manifest in the blob store by its digest.
 func (r *Registry) storeManifestBlob(digest string, data []byte) error {
 	digestHex := strings.TrimPrefix(digest, "sha256:")
-	blobPath := r.paths.OCICacheBlob(digestHex)
 
 	// Verify digest matches
 	actualDigest := computeDigest(data)
@@ -171,7 +171,7 @@ func (r *Registry) storeManifestBlob(digest string, data []byte) error {
 		return fmt.Errorf("digest mismatch: expected %s, got %s", digest, actualDigest)
 	}
 
-	return os.WriteFile(blobPath, data, 0644)
+	return ocicache.WriteBlob(r.paths.SystemOCICache(), v1.Hash{Algorithm: "sha256", Hex: digestHex}, data)
 }
 
 // responseWrapper captures the status code from the response

From 1679ab73ce124a356ac246264dd045569cccf93c Mon Sep 17 00:00:00 2001
From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com>
Date: Fri, 9 Oct 2026 18:32:38 +0000
Subject: [PATCH 5/5] Capture instance ID before registering QEMU test cleanup

waitForInstanceState returns a nil instance on timeout and the tests
reassign inst from it, so the cleanup registered earlier dereferenced nil
and panicked the whole package instead of reporting one failed test.
---
 lib/instances/qemu_test.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go
index 6ee41c90e..6671312b2 100644
--- a/lib/instances/qemu_test.go
+++ b/lib/instances/qemu_test.go
@@ -256,10 +256,11 @@ func TestQEMUMicroVMEndToEnd(t *testing.T) {
 		Hypervisor:     hypervisor.TypeQEMUMicroVM,
 	})
 	require.NoError(t, err)
-	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, inst.Id) })
+	instanceID := inst.Id
+	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instanceID) })
 	require.Equal(t, hypervisor.TypeQEMUMicroVM, inst.HypervisorType)
 
-	inst, err = waitForInstanceState(ctx, manager, inst.Id, StateRunning, integrationTestTimeout(30*time.Second))
+	inst, err = waitForInstanceState(ctx, manager, instanceID, StateRunning, integrationTestTimeout(30*time.Second))
 	require.NoError(t, err)
 	require.NoError(t, waitForQEMUReady(ctx, inst.SocketPath, inst.HypervisorType, integrationTestTimeout(30*time.Second)))
 	assertHostCanReachNginx(t, inst.IP, 80, 30*time.Second)
@@ -302,8 +303,9 @@ func TestQEMUStoppedSnapshotSwitchesToMicroVM(t *testing.T) {
 		Hypervisor:     hypervisor.TypeQEMU,
 	})
 	require.NoError(t, err)
-	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, inst.Id) })
-	inst, err = waitForInstanceState(ctx, manager, inst.Id, StateRunning, integrationTestTimeout(30*time.Second))
+	instanceID := inst.Id
+	t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instanceID) })
+	inst, err = waitForInstanceState(ctx, manager, instanceID, StateRunning, integrationTestTimeout(30*time.Second))
 	require.NoError(t, err)
 
 	inst, err = manager.StopInstance(ctx, inst.Id)