fix(acp): retry a transient shared runtime spawn

docs/system-specs/modules/acp-client.md

@@ -1265,7 +1265,13 @@ The coordinator is keyed by event loop so embedded/test loops never share an
 `asyncio.Semaphore`; cancellation while queued or starting returns the permit,
 and the existing spawn guard still kills a subprocess when initialization is
 cancelled or fails. It uses only asyncio/threading primitives and has no POSIX-only
-behavior.
+behavior. A spawn-level `OSError` gets exactly one retry after two seconds while
+holding its permit: this bridges a Kiro CLI self-update that replaces its executable
+in place without exceeding the cold-start cap. The post-exec shim retries its own
+`execv` on `ENOENT`/`ETXTBSY` (six tries over two seconds), so the same replacement
+window does not surface as an exec failure on the shim path, where the parent's
+spawn has already succeeded and only exit 127 would be left to report it.
+Authentication, protocol, and configuration errors are not retried.
 
 Structured `acp_cold_start` logs distinguish queue wait and spawn, and include
 bounded active/queued counts, outcome, duration, backend class, and coarse process

src/kiro_crew/_spawn_exec_shim.py

@@ -40,8 +40,10 @@
 
 from __future__ import annotations
 
+import errno
 import os
 import sys
+import time
 
 try:
     import resource as _resource
@@ -57,6 +59,14 @@
 # that only sees the exit status can still tell an exec failure from the
 # command's own nonzero exit.
 _EXEC_FAILED = 127
+# A target that raises these from ``execv`` is transiently absent, not broken:
+# the Kiro CLI replaces its own executable during an update, and a spawn landing
+# inside that rename window sees ENOENT or ETXTBSY until the replacement settles.
+_EXECV_RETRYABLE_ERRNOS = (errno.ENOENT, errno.ETXTBSY)
+# Six tries over two seconds ride out that window. Reporting it as exit 127
+# instead would make the parent mark the runtime dead with no retry.
+_EXECV_RETRY_ATTEMPTS = 6
+_EXECV_RETRY_DELAY_S = 0.4
 # Matches sandbox.session_host_preexec: raise NOFILE to the inherited hard cap,
 # or to this floor when the kernel reports no ceiling at all.
 _UNLIMITED_NOFILE_FLOOR = 65536
@@ -288,17 +298,26 @@ def main(argv: list[str] | None = None) -> int:
     _apply_rlimits(pairs)
     if want_oom_bias:
         _bias_oom_score()
-    try:
-        # execv, not execve: the environment this process was given IS the
-        # environment the caller built for the command, and passing it through
-        # untouched avoids rebuilding the whole mapping under the new limits.
-        # No PATH search -- the caller resolves argv[0] so a missing command
-        # surfaces as FileNotFoundError at the spawn, as it did without a shim.
-        os.execv(encoded[0], encoded)
-    except OSError as exc:
-        sys.stderr.write(f"spawn shim: cannot execute {args[0]!r}: {exc.strerror}\n")
-        return _EXEC_FAILED
-    return _EXEC_FAILED  # pragma: no cover - execv does not return on success
+    # execv, not execve: the environment this process was given IS the
+    # environment the caller built for the command, and passing it through
+    # untouched avoids rebuilding the whole mapping under the new limits.
+    # No PATH search -- the caller resolves argv[0] in the parent, so a target
+    # that was already missing at resolve time failed the spawn there.
+    # The retry rides out the CLI self-update rename window, which the
+    # runtime spawn path owns. A terminal (--ctty-fd) command that is already
+    # missing must fail fast instead of stalling the shell for the budget.
+    attempts = 1 if ctty_fd is not None else _EXECV_RETRY_ATTEMPTS
+    for attempt in range(attempts):
+        try:
+            os.execv(encoded[0], encoded)
+        except OSError as exc:
+            if exc.errno in _EXECV_RETRYABLE_ERRNOS and attempt < attempts - 1:
+                time.sleep(_EXECV_RETRY_DELAY_S)
+                continue
+            sys.stderr.write(f"spawn shim: cannot execute {args[0]!r}: {exc.strerror}\n")
+            return _EXEC_FAILED
+        return _EXEC_FAILED  # pragma: no cover - execv does not return on success
+    return _EXEC_FAILED  # pragma: no cover - the loop returns on every attempt
 
 
 if __name__ == "__main__":  # pragma: no cover - exercised as a spawned process

src/kiro_crew/acp/runtime.py

@@ -917,6 +917,9 @@ def _capped_names(names: list[str]) -> str:
 # cost across many background prompts.
 _DEFAULT_MAX_AGE_SECS = 6 * 3600  # 6 hours
 _DEFAULT_MAX_RSS_MB = 500.0  # 500 MiB
+# The Kiro CLI replaces its own executable in place during an update. A spawn
+# that lands in that short window can fail with OSError and succeeds after this delay.
+_ACP_RUNTIME_RESPAWN_BACKOFF_S = 2.0
 
 # Below this uptime the RSS staleness probe is skipped entirely (see
 # _is_stale()). A freshly-(re)used runtime has not had time to grow, so this
@@ -1505,6 +1508,37 @@ class _MirroredSessionMcp(NamedTuple):
     """
 
 
+async def _retrying_spawn_factory(
+    factory: "Callable[..., Awaitable[asyncio.subprocess.Process]]", **kwargs: Any
+) -> asyncio.subprocess.Process:
+    """Drive a subprocess factory, retrying ONE creation failure after a backoff.
+
+    Shaped as the factory
+    :func:`kiro_crew.platform_compat.create_windows_cleanup_owned_process`
+    drives: on Windows that call passes ``windows_cleanup_owner`` down to
+    whatever it invokes, so the keyword has to survive the hop to the bound
+    :func:`create_subprocess_limited`. The Kiro CLI replaces its own executable
+    in place during an update; a spawn that lands in that short window fails
+    with ``OSError`` and succeeds after ``_ACP_RUNTIME_RESPAWN_BACKOFF_S``.
+    Retried before this runtime records process state, so a failed attempt is
+    indistinguishable from never having tried. Never a loop: the exit condition
+    is the CLI finishing its own replacement.
+    """
+    for attempt in range(2):
+        try:
+            return await factory(**kwargs)
+        except OSError as exc:
+            if attempt:
+                raise
+            logger.warning(
+                "ACP runtime subprocess creation failed (%s), retrying after "
+                "adapter replacement window...",
+                exc,
+            )
+            await asyncio.sleep(_ACP_RUNTIME_RESPAWN_BACKOFF_S)
+    raise AssertionError("unreachable: the second attempt returns or re-raises")
+
+
 class AcpRuntime:
     """Owns one kiro-cli acp subprocess with single-reader demux.
 
@@ -2626,33 +2660,36 @@ def _resolve_env_off_loop() -> None:
         try:
             self._process = await platform_compat.create_windows_cleanup_owned_process(
                 functools.partial(
-                    create_subprocess_limited,
-                    *argv,
-                    stdin=asyncio.subprocess.PIPE,
-                    stdout=asyncio.subprocess.PIPE,
-                    stderr=asyncio.subprocess.PIPE,
-                    cwd=self._spawn_work_dir,
-                    limit=_STDOUT_BUFFER_LIMIT,
-                    # POSIX: setsid so kill() can killpg the whole tree. Windows:
-                    # start_new_session is silently ignored; CREATE_NEW_PROCESS_GROUP
-                    # makes the child tree taskkill /T-reapable (see platform_compat
-                    # spawn-isolation note). CREATE_NO_WINDOW suppresses the console
-                    # window Windows would otherwise pop for this console child spawned
-                    # from the windowless gateway (0 on POSIX, so no effect there).
-                    start_new_session=platform_compat.IS_POSIX,
-                    creationflags=(
-                        platform_compat.CREATE_NEW_PROCESS_GROUP
-                        | platform_compat._SUBPROCESS_NO_WINDOW
-                        | platform_compat.CREATE_SUSPENDED
+                    _retrying_spawn_factory,
+                    functools.partial(
+                        create_subprocess_limited,
+                        *argv,
+                        stdin=asyncio.subprocess.PIPE,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.PIPE,
+                        cwd=self._spawn_work_dir,
+                        limit=_STDOUT_BUFFER_LIMIT,
+                        # POSIX: setsid so kill() can killpg the whole tree. Windows:
+                        # start_new_session is silently ignored; CREATE_NEW_PROCESS_GROUP
+                        # makes the child tree taskkill /T-reapable (see platform_compat
+                        # spawn-isolation note). CREATE_NO_WINDOW suppresses the console
+                        # window Windows would otherwise pop for this console child spawned
+                        # from the windowless gateway (0 on POSIX, so no effect there).
+                        start_new_session=platform_compat.IS_POSIX,
+                        creationflags=(
+                            platform_compat.CREATE_NEW_PROCESS_GROUP
+                            | platform_compat._SUBPROCESS_NO_WINDOW
+                            | platform_compat.CREATE_SUSPENDED
+                        ),
+                        # None off macOS, where nothing binds. When set, the child enters
+                        # the workspace through this verified descriptor instead of
+                        # resolving ``cwd``'s pathname, which a same-UID symlink retarget
+                        # could aim elsewhere in between; ``cwd`` stays the same directory
+                        # by name so the spawn keeps reporting a real path.
+                        chdir_fd=self._bound_workspace_fd,
+                        env=env,
+                        profile=RLIMIT_PROFILE_SESSION_HOST,
                     ),
-                    # None off macOS, where nothing binds. When set, the child enters
-                    # the workspace through this verified descriptor instead of
-                    # resolving ``cwd``'s pathname, which a same-UID symlink retarget
-                    # could aim elsewhere in between; ``cwd`` stays the same directory
-                    # by name so the spawn keeps reporting a real path.
-                    chdir_fd=self._bound_workspace_fd,
-                    env=env,
-                    profile=RLIMIT_PROFILE_SESSION_HOST,
                 ),
             )
         except BaseException:

test/test_acp_runtime.py

@@ -1762,6 +1762,21 @@ async def fail_spawn(self):
     assert admission.active == 0
 
 
+@pytest.mark.asyncio
+async def test_runtime_spawn_retries_one_creation_failure_after_backoff(monkeypatch):
+    import kiro_crew.acp.runtime as runtime_mod
+
+    process = MagicMock()
+    sleep = AsyncMock()
+    create = AsyncMock(side_effect=[FileNotFoundError("kiro-cli is being replaced"), process])
+    monkeypatch.setattr(runtime_mod.asyncio, "sleep", sleep)
+
+    assert await runtime_mod._retrying_spawn_factory(create) is process
+
+    assert create.await_count == 2
+    sleep.assert_awaited_once_with(runtime_mod._ACP_RUNTIME_RESPAWN_BACKOFF_S)
+
+
 def test_cold_start_admission_registry_releases_contended_closed_loop(monkeypatch):
     import kiro_crew.acp.runtime as runtime_mod
 

test/test_spawn_exec_shim.py

@@ -109,16 +109,57 @@ def test_empty_command_is_refused(self, capsys):
         assert shim.main(["--"]) == 127
         assert "no command" in capsys.readouterr().err
 
-    def test_exec_failure_reports_127_not_a_traceback(self, capsys):
+    def test_exec_failure_reports_127_not_a_traceback(self, capsys, monkeypatch):
+        sleeps: list[float] = []
+        monkeypatch.setattr(shim.time, "sleep", sleeps.append)
         assert shim.main(["--", "/nonexistent/binary"]) == 127
         assert "cannot execute" in capsys.readouterr().err
+        assert sleeps == [shim._EXECV_RETRY_DELAY_S] * (shim._EXECV_RETRY_ATTEMPTS - 1)
+
+    def test_transient_target_absence_is_retried_but_other_errors_are_not(
+        self, capsys, monkeypatch
+    ):
+        calls: list[tuple] = []
+        sleeps: list[float] = []
+        monkeypatch.setattr(shim.time, "sleep", sleeps.append)
+
+        def fake_execv(path, argv):
+            calls.append((path, argv))
+            if len(calls) == 1:
+                raise FileNotFoundError(2, "No such file or directory")
+            raise OSError(13, "Permission denied")
+
+        with patch.object(shim.os, "execv", fake_execv):
+            assert shim.main(["--", "/bin/echo"]) == 127
+        # One retry for the transient miss, then EACCES stops the loop.
+        assert len(calls) == 2
+        assert sleeps == [shim._EXECV_RETRY_DELAY_S]
+        assert "Permission denied" in capsys.readouterr().err
+
+    @posix_only
+    def test_a_terminal_command_missing_is_not_retried(self, capsys, monkeypatch):
+        """The rename-window retry is the runtime spawn's; a terminal command fails fast."""
+        calls: list[tuple] = []
+        sleeps: list[float] = []
+        monkeypatch.setattr(shim.time, "sleep", sleeps.append)
+        monkeypatch.setattr(shim.os, "login_tty", lambda fd: None)
+
+        def fake_execv(path, argv):
+            calls.append((path, argv))
+            raise OSError(2, "No such file or directory")
+
+        with patch.object(shim.os, "execv", fake_execv):
+            assert shim.main(["--ctty-fd=0", "--", "/bin/true"]) == 127
+        assert len(calls) == 1
+        assert sleeps == []
+        assert "cannot execute" in capsys.readouterr().err
 
     def test_separator_inside_the_command_is_not_consumed(self):
         calls: list[list[bytes]] = []
 
         def fake_execv(_path, argv):
             calls.append(argv)
-            raise OSError(2, "stop here")
+            raise OSError(13, "stop here")
 
         with patch.object(shim.os, "execv", fake_execv):
             shim.main(["--", "/bin/echo", "--", "--rlimits=bogus"])
@@ -133,7 +174,7 @@ def test_limits_are_applied_before_exec(self):
             patch.object(
                 shim.os,
                 "execv",
-                lambda *_a: order.append("exec") or (_ for _ in ()).throw(OSError(2, "x")),
+                lambda *_a: order.append("exec") or (_ for _ in ()).throw(OSError(13, "x")),
             ),
         ):
             shim.main(["--rlimits=RLIMIT_NOFILE:1024", "--oom-bias", "--", "/bin/true"])
@@ -169,7 +210,7 @@ def test_the_bound_directory_is_entered_before_the_limits(self):
             patch.object(
                 shim.os,
                 "execv",
-                lambda *_a: order.append("exec") or (_ for _ in ()).throw(OSError(2, "x")),
+                lambda *_a: order.append("exec") or (_ for _ in ()).throw(OSError(13, "x")),
             ),
         ):
             shim.main(["--rlimits=RLIMIT_NOFILE:1024", "--chdir-fd=9", "--", "/bin/true"])
@@ -195,14 +236,35 @@ def test_oom_bias_only_when_requested(self):
         biased: list[bool] = []
         with (
             patch.object(shim, "_bias_oom_score", lambda: biased.append(True)),
-            patch.object(shim.os, "execv", lambda *_a: (_ for _ in ()).throw(OSError(2, "x"))),
+            patch.object(shim.os, "execv", lambda *_a: (_ for _ in ()).throw(OSError(13, "x"))),
         ):
             shim.main(["--", "/bin/true"])
             assert biased == []
             shim.main(["--oom-bias", "--", "/bin/true"])
             assert biased == [True]
 
 
+@posix_only
+class TestShimRetryEndToEnd:
+    @pytest.mark.asyncio
+    async def test_target_appearing_inside_the_retry_window_is_executed(self, tmp_path):
+        """The defect: the CLI binary is briefly absent mid-spawn, then reappears."""
+        target = tmp_path / "late-bird"
+
+        def create_target() -> None:
+            target.write_text("#!/bin/sh\nexit 0\n")
+            target.chmod(0o755)
+
+        threading.Timer(0.3, create_target).start()
+        proc = await asyncio.create_subprocess_exec(*spawn_shim_argv(), str(target))
+        try:
+            rc = await asyncio.wait_for(proc.wait(), timeout=30)
+        finally:
+            if proc.returncode is None:
+                proc.kill()
+        assert rc == 0
+
+
 # --------------------------------------------------------------------------
 # Parent side: the argv prefix and the profiles
 # --------------------------------------------------------------------------