fix(proxy): recover tool-complete sessions from unavailable owners

app/core/errors.py

@@ -90,6 +90,11 @@ class ResponseFailedEvent(TypedDict):
     _codex_lb_synthetic_transport_failure: NotRequired[bool]
 
 
+PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE = (
+    "Previous response owner account is unavailable. Retry with complete account-neutral history "
+    "without previous_response_id, or start a new session."
+)
+
 PREVIOUS_RESPONSE_STREAM_INCOMPLETE_MESSAGE = "Upstream websocket closed before response.completed"
 # Local bridge recovery (fresh replay, context-overflow rollover, previous
 # response rebind) tears down our own upstream session. It is not an upstream

app/modules/proxy/_service/compact.py

@@ -25,7 +25,7 @@
 from app.core.clients.proxy import compact_responses as core_compact_responses
 from app.core.config.settings import get_settings
 from app.core.config.settings_cache import get_settings_cache
-from app.core.errors import openai_error
+from app.core.errors import PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE, openai_error
 from app.core.openai.exceptions import ClientPayloadError
 from app.core.openai.models import CompactResponsePayload
 from app.core.openai.requests import ResponsesCompactRequest
@@ -915,7 +915,7 @@ async def settle_compact_usage(
                     else None,
                 )
                 if len(selection_inputs.accounts) != 1:
-                    message = "Previous response owner account is unavailable; retry later."
+                    message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
                     _record_continuity_fail_closed(
                         surface="compact",
                         reason="owner_account_unavailable",

app/modules/proxy/_service/http_bridge/helpers.py

@@ -44,6 +44,7 @@
 from app.core.errors import (
     HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE,
     HTTP_BRIDGE_LOCAL_RESET_MESSAGE,
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     OpenAIErrorDetail,
     OpenAIErrorEnvelope,
     OpenAIErrorParam,
@@ -3435,7 +3436,7 @@ def _http_bridge_previous_response_owner_unavailable_error() -> ProxyResponseErr
         502,
         openai_error(
             "previous_response_owner_unavailable",
-            "Previous response owner account is unavailable; retry later.",
+            PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
             error_type="server_error",
         ),
     )

app/modules/proxy/_service/http_bridge/streaming.py

@@ -38,6 +38,7 @@
 from app.core.clients.proxy_websocket import UpstreamWebSocketTransportError
 from app.core.clock import Clock, Scheduler, clock_for, scheduler_for
 from app.core.errors import (
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     PREVIOUS_RESPONSE_STREAM_INCOMPLETE_MESSAGE,
     openai_error,
     response_failed_event,
@@ -2194,7 +2195,7 @@ def switch_to_account_neutral_replay(
                 502,
                 openai_error(
                     "previous_response_owner_unavailable",
-                    "Previous response owner account is unavailable; retry later.",
+                    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
                 ),
             )
             _record_continuity_fail_closed(

app/modules/proxy/_service/streaming/helpers.py

@@ -39,6 +39,7 @@
 )
 from app.core.errors import (
     PREVIOUS_RESPONSE_MALFORMED_PARAM_REASON,
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     PREVIOUS_RESPONSE_STREAM_INCOMPLETE_MESSAGE,
     SYNTHETIC_TRANSPORT_FAILURE_CODES,
     OpenAIErrorParam,
@@ -665,7 +666,7 @@ def _rewrite_previous_response_stream_error(
         )
         return (
             "previous_response_owner_unavailable",
-            "Previous response owner account is unavailable; retry later.",
+            PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
             normalized_code,
         )
     return None

app/modules/proxy/_service/streaming/retry.py

@@ -27,6 +27,7 @@
 )
 from app.core.clock import REAL_CLOCK, REAL_SCHEDULER, Clock, Scheduler, clock_for, scheduler_for
 from app.core.errors import (
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     SYNTHETIC_TRANSPORT_FAILURE_CODES,
     openai_error,
     synthetic_transport_failure_event,
@@ -1288,7 +1289,7 @@ async def _retry_account_model_rejection(
                         account_ids=None,
                     )
                     if len(selection_inputs.accounts) != 1:
-                        message = "Previous response owner account is unavailable; retry later."
+                        message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
                         _record_continuity_fail_closed(
                             surface="http_stream",
                             reason="owner_account_unavailable",
@@ -1717,7 +1718,7 @@ async def _retry_account_model_rejection(
                         return
                     if require_preferred_account and preferred_account_id is not None:
                         error_code = "previous_response_owner_unavailable"
-                        message = "Previous response owner account is unavailable; retry later."
+                        message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
                         reason = "owner_account_unavailable"
                         upstream_error_code = "no_accounts"
                         if selection.error_code == "continuity_owner_conflict":
@@ -1868,7 +1869,7 @@ async def _retry_account_model_rejection(
                         )
                     else:
                         error_code = "previous_response_owner_unavailable"
-                        message = "Previous response owner account is unavailable; retry later."
+                        message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
                         reason = "owner_account_unavailable"
                         upstream_error_code = "upstream_unavailable"
                         if selection.error_code == "continuity_owner_conflict":

app/modules/proxy/_service/websocket/helpers.py

@@ -44,6 +44,7 @@
     PREVIOUS_RESPONSE_MALFORMED_PARAM_REASON,
     PREVIOUS_RESPONSE_NOT_FOUND_CODE,
     PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE,
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     PREVIOUS_RESPONSE_STREAM_INCOMPLETE_MESSAGE,
     OpenAIErrorEnvelope,
     OpenAIErrorParam,
@@ -1634,7 +1635,7 @@ def _rewrite_websocket_previous_response_owner_unavailable_event(
     )
     rewritten_event_payload = response_failed_event(
         "upstream_unavailable",
-        "Previous response owner account is unavailable; retry later.",
+        PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
         error_type="server_error",
         response_id=_websocket_downstream_response_id(request_state),
     )

app/modules/proxy/_service/websocket/mixin.py

@@ -67,6 +67,7 @@
 from app.core.errors import (
     PREVIOUS_RESPONSE_MALFORMED_PARAM_REASON,
     PREVIOUS_RESPONSE_NOT_FOUND_CODE,
+    PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE,
     STREAM_INCOMPLETE_ANCHOR_NEUTRAL_MESSAGES,
     OpenAIErrorEnvelope,
     OpenAIErrorParam,
@@ -4035,7 +4036,7 @@ async def _heartbeat(remaining_seconds: float) -> None:
             and account.id != preferred_account_id
         ):
             await proxy._load_balancer.release_account_lease(selection.lease)
-            message = "Previous response owner account is unavailable; retry later."
+            message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
             _record_continuity_fail_closed(
                 surface="websocket_connect",
                 reason="owner_account_unavailable",
@@ -4113,7 +4114,7 @@ async def _heartbeat(remaining_seconds: float) -> None:
                     error_message=error_message,
                 )
                 return None
-            message = "Previous response owner account is unavailable; retry later."
+            message = PREVIOUS_RESPONSE_OWNER_UNAVAILABLE_MESSAGE
             _record_continuity_fail_closed(
                 surface="websocket_connect",
                 reason="owner_account_unavailable",

app/modules/proxy/replay_safety.py

@@ -423,6 +423,18 @@ def responses_input_suffix_matches_pending_tool_calls(
         and _fresh_developer_interleave_is_bounded(suffix, index=1)
     ):
         suffix = [suffix[0], suffix[2]]
+    # Fresh user input may follow a complete manifest, but cannot replace
+    # missing results or relax the canonical ownership/known-fields checks.
+    first_followup = next(
+        (index for index, item in enumerate(suffix) if isinstance(item, dict) and _is_fresh_followup_input(item)),
+        len(suffix),
+    )
+    followup = suffix[first_followup:]
+    if not all(isinstance(item, dict) and _is_fresh_followup_input(item) for item in followup):
+        return False
+    if not responses_input_items_are_self_contained_fresh_replay(followup):
+        return False
+    suffix = suffix[:first_followup]
     if not all(
         isinstance(item, dict)
         and isinstance(item.get("type"), str)

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-09-09

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/context.md

@@ -0,0 +1,13 @@
+# Recovery evidence and operational context
+
+Issue #27 was reproduced on both public HTTP endpoints with a rate-limited owner: the full request contained a durable-prefix match, a completed custom tool call/result and a fresh goal instruction, yet returned 502 `previous_response_owner_unavailable`. The old manifest helper rejected the trailing user message.
+
+The implementation extends that proof only after the complete batch. It validates new input using both fresh-input content checks and the canonical allowed-fields/ownership validator. It keeps the existing full-body projection, exact manifest, account scope, file pins and pre-dispatch recovery. The existing session recovery machinery retires the old anchor and records the replacement; no new owner store is introduced.
+
+For example, owner A finishes a shell tool call, reaches its five-hour limit, and the client resends the full history with the result and a new goal instruction. The request goes to eligible B without A's response ID and later anchored turns continue on B. An A-owned file, missing tool result or unknown ownership field prevents replay; three repeated requests remain pre-dispatch and never call the retry-circuit failure writer.
+
+Verification: before-fix public regression failed on both HTTP endpoints; 552 replay/HTTP/WebSocket tests pass; all 32 endpoint/rate-or-quota/explicit-anchor/safe-or-rejected recovery scenarios pass; 377 owner/continuity tests pass. Ruff, type checking, architecture, timing seams, cancellation safety, strict change validation and all 63 main spec validations pass. The 32-case run includes literal checks for actionable error guidance, not only a comparison against the implementation constant.
+
+Regression scenarios were adapted from upstream PR #2121 and expanded. Its reported concern about relaxed validation of trailing input is covered by canonical ownership validation and explicit unknown-field/owner-metadata tests. Opaque compaction remains nonportable. Direct WebSocket behavior keeps its existing ownership policy and stable error code while gaining the same actionable guidance.
+
+PR #30 follows beta6 integration PR #31; beta5 integration PR #29 is merged. All 1,660 replay-safety, HTTP bridge and direct WebSocket tests passed after incorporating beta6. Final-head GitHub checks and clean mergeability remain deployment/merge gates; local verification does not replace them.

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/design.md

@@ -0,0 +1,15 @@
+## Context
+
+The existing HTTP bridge validates a durable prefix fingerprint and retains either a prior assistant response or the exact recorded pending-tool manifest. Its account-neutral projection drops owner-bound reasoning only when enough plaintext context remains. The manifest checker currently requires every suffix item to be a tool call/result, so a following user instruction incorrectly invalidates complete context.
+
+## Decisions
+
+Split a suffix at its first fresh user input. Require every following item to be fresh input and validate it with the canonical self-contained replay classifier, including allowed fields and ownership metadata. Validate the earlier tool batch exactly against the durable manifest, preserving ordering, type, duplicate, collision and missing-result checks. Keep the existing bounded developer-interleave exception unchanged; it cannot be extended with a trailing suffix.
+
+Reuse existing prefix proof, complete-body projection, account scope, file affinity, pre-dispatch gates, replacement session identity and durable ownership fencing. No new replay registry or broad abandonment of explicit continuity. Explicit previous-response references may be removed only where existing full-context proof authorizes it.
+
+Keep the existing owner-unavailable error code but explain that a complete account-neutral resend without the old anchor, or a new session, is required when recovery cannot be proven. Verify HTTP and direct-WebSocket error contracts and that pre-dispatch rejection does not consume the retry circuit.
+
+## Constraints and failure modes
+
+Fresh input must not hide a missing parallel result or introduce unknown account-owned metadata. Opaque encrypted compaction, conversation references, file ownership, incomplete/mismatched prefixes and in-flight ambiguous outcomes remain fail-closed. A quota-limited owner A with a complete proven tool result plus a new goal instruction can recover to eligible B; a file owned by A cannot.

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/proposal.md

@@ -0,0 +1,20 @@
+## Why
+
+Issue #27 describes existing Codex sessions stranded on a rate-limited owner while another compatible account can serve them. Upstream already projects proven full context into an account-neutral replay, but rejects a complete tool batch followed by fresh user input even when durable metadata proves the batch complete. Unsafe requests also receive only an instruction to retry later.
+
+## What Changes
+
+- Admit fresh user input after an exact, complete durable tool manifest, preserving strict whole-body account-neutral validation and existing owner fencing.
+- Preserve calls, results and new input in the replay; remove the old account-bound anchor through the existing recovery path and continue on an eligible replacement.
+- Make unreplayable owner-unavailable errors actionable while retaining stable error codes.
+- Prove public HTTP recovery on a rate/usage-limited owner, continued replacement ownership, unsafe replay rejection and bounded repeated failures.
+
+## Capabilities
+
+### Modified Capabilities
+
+- `responses-api-compat`: prove a complete tool batch with trailing fresh input and provide actionable unreplayable-owner errors.
+
+## Impact
+
+Replay proof, continuity error text and regression coverage. No settings, schema, dashboard or default changes. Refs #27 and upstream #1707/#2121. Opaque compaction or account-owned files remain nonportable.

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/specs/responses-api-compat/spec.md

@@ -0,0 +1,28 @@
+## ADDED Requirements
+
+### Requirement: Complete tool context permits a fresh user follow-up during owner recovery
+
+When a Responses continuation has a fingerprint-verified durable input prefix and every call and matching result in the recorded prior-response tool manifest, the replay proof MUST permit a trailing self-contained user-input suffix. The proof MUST validate the complete suffix against the canonical account-neutral allowed-fields and content contract and MUST preserve exact call IDs, call types, ordering, complete results and collision checks. It MUST reject user input interleaved with incomplete results and any subsequent call, result, assistant or developer item. The existing bounded developer-interleave exception MUST NOT gain a trailing-input extension.
+
+Cross-account recovery MUST still require full context, account-neutral projection, existing account scope and file ownership checks, and safe pre-dispatch state. Recovery MUST preserve all proven calls, results and new input, remove the unavailable owner's upstream anchor through the existing fenced recovery path, and allow the client session to continue on the eligible replacement. Explicit previous-response references alone MUST NOT authorize replay.
+
+#### Scenario: Complete batch resumes on another account
+- **GIVEN** an existing session's owner becomes temporarily rate/usage limited and another compatible account is eligible
+- **AND** a full resend matches the stored prefix and completes the exact recorded tool batch before appending fresh user input
+- **WHEN** the complete projected request is account-neutral and safe to dispatch
+- **THEN** the bridge replays it without the old anchor on the replacement
+- **AND** a later anchored continuation stays on that replacement
+
+#### Scenario: Fresh input cannot conceal missing or account-owned context
+- **WHEN** the input omits a recorded tool result, introduces a duplicate or mismatched call, interleaves user input, or contains account-owned files, unknown ownership fields or opaque compaction state
+- **THEN** the proof MUST reject cross-account replay
+
+### Requirement: Unrecoverable owner failures give actionable continuity guidance
+
+An unavailable-owner request whose safe replay cannot be proven MUST fail with its stable continuity error code and guidance to resend complete account-neutral history without the old response anchor or start a new session. A pre-dispatch proof rejection MUST NOT submit to another account or count as an upstream transport failure that opens a poisoned-anchor retry circuit.
+
+#### Scenario: Repeating an unsafe full resend fails without poisoning the retry circuit
+- **GIVEN** an unavailable owner and an unsafe or incomplete replay
+- **WHEN** the client repeats the request
+- **THEN** each request fails with actionable continuity guidance before upstream submission
+- **AND** the rejection does not create an upstream-timeout cooldown

openspec/changes/archive/2026-09-09-recover-unavailable-continuity-owner/tasks.md

@@ -0,0 +1,11 @@
+## 1. Reproduction and implementation
+
+- [x] 1.1 Reproduce the full tool-batch plus user-input failure through both public HTTP endpoints.
+- [x] 1.2 Extend manifest proof without weakening complete-body validation or developer-interleave limits.
+- [x] 1.3 Provide actionable owner-unavailable errors and verify bounded, unpoisoned failure behavior.
+
+## 2. Verification
+
+- [x] 2.1 Verify rate/usage-limited owner recovery, replacement continuity, explicit-anchor proof, file pins and malformed/incomplete replay rejection.
+- [x] 2.2 Run relevant unit/HTTP/WebSocket, lint/type and strict OpenSpec validation.
+- [x] 2.3 Verify OpenSpec against implementation and regression evidence; publish PR #30 fixing #27.

openspec/specs/responses-api-compat/context.md

@@ -266,6 +266,12 @@ OpenSpec change first.
 - Post-deploy: monitor `previous_response_not_found` on `/backend-api/codex/responses`; recurring spikes show repeated continuity failures, which may come from malformed client identifiers, server-side invalidation, or connection lifecycle. Clients should perform the documented full-context retry without `previous_response_id`. Investigate socket-lifecycle remediation only when a separate close-reason, reconnect, or transport diagnostic correlates with the failures.
 - Websocket/Codex CLI tier verification runbook: `openspec/specs/responses-api-compat/ops.md`
 
+## Tool-complete owner recovery
+
+When a full resend matches its durable prefix and settles every recorded tool call, a trailing user instruction is now replayable through the existing account-neutral recovery path. The complete tool batch and new input are retained; stored account-bound anchors are removed only after proof. For example, a rate-limited owner can yield a fully replayable goal continuation to another eligible account, and subsequent anchored turns stay on the replacement.
+
+Missing results, account-owned files, unknown ownership fields and opaque compaction still prevent migration. An owner-unavailable error now explains that the client must send complete account-neutral history without the old response ID or start a new session. This does not make encrypted checkpoints portable. See the normative requirements in [spec.md](spec.md).
+
 
 ## HTTP continuation promotion