diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 2793119189..3a4476ae3d 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -10,6 +10,7 @@ BaseModel, ConfigDict, Field, + ModelWrapValidatorHandler, PrivateAttr, SerializeAsAny, SkipValidation, @@ -640,6 +641,14 @@ class ResponsesTextControls(BaseModel): format: ResponsesTextFormat | None = None +@dataclass(frozen=True, slots=True) +class _ResponsesRawInputProvenance: + raw_input: str | list[JsonValue] + normalized_value: JsonValue + original_instructions: str + normalized_instructions: str + + class ResponsesRequest(BaseModel): model_config = ConfigDict(extra="allow") _codex_lb_client_reasoning_effort: str | None = PrivateAttr(default=None) @@ -650,6 +659,33 @@ class ResponsesRequest(BaseModel): # subscription-overflow dispatch (anchor rule and the source-direction # body); the ChatGPT-bound serialization never consults it. _codex_lb_client_store: bool | None = PrivateAttr(default=None) + _codex_lb_raw_input_provenance: _ResponsesRawInputProvenance | None = PrivateAttr(default=None) + _codex_lb_input_shape_wire_version: str | None = PrivateAttr(default="2") + _codex_lb_legacy_owner_forwarding_input_shape: bool = PrivateAttr(default=False) + + @model_validator(mode="wrap") + @classmethod + def _capture_raw_input_string_provenance( + cls, + data: JsonValue | ResponsesRequest, + handler: ModelWrapValidatorHandler[ResponsesRequest], + ) -> ResponsesRequest: + if isinstance(data, cls): + return handler(data) + raw_input = data.get("input") if is_json_mapping(data) else None + raw_instructions = data.get("instructions") if is_json_mapping(data) else None + request = handler(data) + request._codex_lb_raw_input_provenance = ( + _ResponsesRawInputProvenance( + raw_input=raw_input, + normalized_value=request.input, + original_instructions=raw_instructions if isinstance(raw_instructions, str) else "", + normalized_instructions=request.instructions, + ) + if isinstance(raw_input, str) or is_json_list(raw_input) + else None + ) + return request @model_validator(mode="before") @classmethod @@ -750,10 +786,10 @@ def model_dump_for_forwarding(self) -> MutableJsonObject: Like ``model_dump(mode="json", exclude_none=True)`` but without synthesizing fields the client never sent. Used by every path that - forwards this request as a JSON body — the multi-instance owner - forward (``HTTPBridgeOwnerClient``) and model-source Responses - egress — so that field omission survives the hop and the receiving - side does not re-mark ``tools`` as explicitly set. + forwards this request to a model source so that field omission + survives the hop and the receiver does not re-mark ``tools`` as + explicitly set. The multi-instance owner wire has a dedicated dump + because it must also preserve pre-validation input shape. """ payload: MutableJsonObject = self.model_dump(mode="json", exclude_none=True) if "tools" not in self.model_fields_set: @@ -770,6 +806,23 @@ def model_dump_for_forwarding(self) -> MutableJsonObject: payload.pop("tools", None) return payload + def model_dump_for_http_bridge_owner_forwarding(self) -> MutableJsonObject: + """Dump the exact request body posted to an internal bridge owner. + + Request validation normalizes a client string into one input item. + Preserve the original string on this internal hop so the owner can + revalidate it without changing the full-resend boundary. Keep model + source forwarding normalized; only the internal owner needs to repeat + local request-shape classification. + """ + payload = self.model_dump_for_forwarding() + provenance = self._codex_lb_raw_input_provenance + if provenance is not None and self.input is provenance.normalized_value: + payload["input"] = provenance.raw_input + if self.instructions == provenance.normalized_instructions: + payload["instructions"] = provenance.original_instructions + return payload + def to_payload(self) -> JsonObject: payload = _strip_unsupported_fields(self.model_dump_for_forwarding()) _normalize_compaction_trigger_singleton(payload) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index a6900de0d3..95ee2c41d1 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -71,6 +71,7 @@ ) from app.core.resilience.overload import local_overload_error from app.core.types import JsonValue +from app.core.utils.json_guards import is_json_list, is_json_mapping from app.core.utils.request_id import get_request_id from app.core.utils.shared_future import _await_task_deferring_cancellation, wait_on_shared_future from app.core.utils.sse import format_sse_event, parse_sse_data_json @@ -131,6 +132,7 @@ ) from app.modules.proxy._service.support import ( _HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401 + _PENDING_TOOL_CALL_OUTPUT_ITEM_TYPES, _REQUEST_TRANSPORT_HTTP, _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 _http_bridge_session_supports_service_tier, @@ -2078,20 +2080,73 @@ def _http_bridge_session_retiring_with_visible_requests(session: "_HTTPBridgeSes def _http_bridge_payload_looks_like_full_resend(payload: ResponsesRequest) -> bool: - input_value = payload.input + raw_input_provenance = payload._codex_lb_raw_input_provenance + if raw_input_provenance is not None and payload.input is raw_input_provenance.normalized_value: + input_value = raw_input_provenance.raw_input + else: + input_value = payload.input + if isinstance(input_value, str): return len(input_value) >= 4096 if isinstance(input_value, Sequence) and not isinstance(input_value, (str, bytes, bytearray)): if len(input_value) > 1: - return True + if payload._codex_lb_legacy_owner_forwarding_input_shape: + return True + # Multiple outputs can be one delta-only continuation for + # parallel calls completed by the anchored response. Without the + # corresponding call items this payload is not self-contained and + # must keep its durable anchor. + return not all( + _http_bridge_input_item_type(item) in _PENDING_TOOL_CALL_OUTPUT_ITEM_TYPES for item in input_value + ) if len(input_value) == 1: + if ( + not payload._codex_lb_legacy_owner_forwarding_input_shape + and _http_bridge_input_item_type(input_value[0]) in _PENDING_TOOL_CALL_OUTPUT_ITEM_TYPES + ): + return False + if payload._codex_lb_legacy_owner_forwarding_input_shape: + normalized_text_length = _http_bridge_legacy_normalized_input_text_length(input_value) + if normalized_text_length is not None: + return normalized_text_length >= 4096 + # Pre-classifier owners measured a one-item array's item, + # rather than the array envelope. Keep that predicate for + # genuinely array-shaped legacy forwards so a large item is + # not silently downgraded to a delta. + try: + return len(json.dumps(input_value[0], ensure_ascii=True, separators=(",", ":"))) >= 4096 + except (OverflowError, TypeError, ValueError): + return False try: - return len(json.dumps(input_value[0], ensure_ascii=True, separators=(",", ":"))) >= 4096 - except TypeError: + return len(json.dumps(input_value, ensure_ascii=True, separators=(",", ":"))) >= 4096 + except (OverflowError, TypeError, ValueError): return False return False +def _http_bridge_legacy_normalized_input_text_length(input_value: Sequence[JsonValue]) -> int | None: + """Recognize the canonical one-item array produced from a raw string. + + A legacy owner receives the normalized array without the original string + provenance. Its envelope is implementation detail, so classify this + exact shape by the contained text length and use the legacy item-size + rule only for genuinely array-shaped inputs. + """ + if len(input_value) != 1 or not is_json_mapping(input_value[0]): + return None + item = input_value[0] + if set(item) != {"role", "content"} or item.get("role") != "user": + return None + content = item.get("content") + if not is_json_list(content) or len(content) != 1 or not is_json_mapping(content[0]): + return None + part = content[0] + if set(part) != {"type", "text"} or part.get("type") != "input_text": + return None + text = part.get("text") + return len(text) if isinstance(text, str) else None + + def _preferred_http_bridge_reconnect_turn_state(session: "_HTTPBridgeSession") -> str | None: if ( session.codex_session @@ -2547,6 +2602,15 @@ def _durable_bridge_lookup_active_owner(lookup: DurableBridgeLookup | None) -> s return lookup.owner_instance_id +def _owner_process_epoch( + lookup: DurableBridgeLookup | None, + owner_instance: str, +) -> str | None: + if lookup is None or lookup.owner_instance_id != owner_instance: + return None + return lookup.owner_process_epoch + + def _durable_bridge_lookup_allows_local_reuse( lookup: DurableBridgeLookup | None, *, @@ -3238,13 +3302,15 @@ def _http_bridge_turn_state_anchor_for_owner_failure( headers: Mapping[str, str], previous_response_id: str | None, ) -> str | None: - """Return the turn-state anchor when an owner forward failed unreachable.""" + """Return turn state for an unreachable owner or typed pre-dispatch shape failure.""" if previous_response_id is not None: return None turn_state = _sticky_key_from_turn_state_header(headers) if turn_state is None: return None + if exc.failure_phase == "owner_forward" and exc.failure_detail == "owner_input_shape_upgrade_required": + return turn_state payload = exc.payload if not isinstance(payload, dict): return None @@ -3455,6 +3521,8 @@ def _http_bridge_reconnect_connect_failure( def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool: + if exc.failure_phase == "owner_forward" and exc.failure_detail == "owner_input_shape_upgrade_required": + return True payload = exc.payload if not isinstance(payload, dict): return False @@ -3599,6 +3667,8 @@ def _http_bridge_should_attempt_local_bootstrap_rebind( return False if _sticky_key_from_turn_state_header(headers) is not None: return False + if exc.failure_phase == "owner_forward" and exc.failure_detail == "owner_input_shape_upgrade_required": + return True payload = exc.payload if not isinstance(payload, dict): return False diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 6f3701aa76..8a13ffbf36 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -47,10 +47,7 @@ ) from app.core.utils.locks import fast_lock from app.core.utils.request_id import ensure_request_scope_id -from app.db.models import ( - DashboardSettings, - StickySessionKind, -) +from app.db.models import DashboardSettings, StickySessionKind from app.modules.api_keys.service import ( ApiKeyData, ApiKeyRequestUsageBudget, @@ -116,6 +113,7 @@ _log_http_bridge_event, _log_http_bridge_startup_wait_timeout, _mark_http_bridge_reader_handoff_reconnect_failed, + _owner_process_epoch, _persist_http_bridge_replacement_account, _persistent_http_bridge_affinity, _plan_http_bridge_lru_capacity_closes, @@ -1056,6 +1054,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: owner_instance=owner_instance, owner_endpoint=owner_endpoint, key=key, + owner_process_epoch=_owner_process_epoch(durable_lookup, owner_instance), ) else: if _http_bridge_has_durable_recovery_anchor( diff --git a/app/modules/proxy/_service/http_bridge/owner_forwarding.py b/app/modules/proxy/_service/http_bridge/owner_forwarding.py index 8fbdc032fa..1f3efd78c3 100644 --- a/app/modules/proxy/_service/http_bridge/owner_forwarding.py +++ b/app/modules/proxy/_service/http_bridge/owner_forwarding.py @@ -2,6 +2,7 @@ import asyncio import logging +from dataclasses import replace from enum import StrEnum from typing import Any, AsyncIterator, Mapping, TypeVar @@ -155,7 +156,9 @@ from app.modules.proxy.http_bridge_forwarding import ( HTTPBridgeForwardContext, OwnerForwardRelayFailure, + _http_bridge_owner_forward_requires_shape_upgrade, ) +from app.modules.proxy.ring_membership import HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY logger = logging.getLogger("app.modules.proxy.service") T = TypeVar("T") @@ -476,6 +479,30 @@ def owner_response_rejected() -> None: if api_key_reservation is not None: _signal_propagated_responses_owner_forward_rejected() + owner_supports_input_shape_classifier = False + if ( + _http_bridge_owner_forward_requires_shape_upgrade(payload) + and self._ring_membership is not None + and owner_forward.owner_process_epoch is not None + ): + try: + owner_supports_input_shape_classifier = await self._ring_membership.owner_supports_capability( + owner_forward.owner_instance, + owner_process_epoch=owner_forward.owner_process_epoch, + capability=HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, + ) + except Exception: + logger.debug( + "Failed to prove HTTP bridge owner input-shape capability", + exc_info=True, + ) + + if owner_supports_input_shape_classifier: + forward_context = replace( + forward_context, + expected_owner_process_epoch=owner_forward.owner_process_epoch, + ) + try: async for event_block in self._http_bridge_owner_client.stream_responses( owner_endpoint=owner_forward.owner_endpoint, @@ -483,6 +510,7 @@ def owner_response_rejected() -> None: headers=forward_headers, context=forward_context, request_started_at=request_started_at, + owner_supports_input_shape_classifier=owner_supports_input_shape_classifier, on_request_dispatched=owner_request_dispatched, on_response_rejected=owner_response_rejected, on_response_wait=_signal_propagated_capacity_startup_wait, diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index ec0bbbeeac..7c56c2c0b6 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -374,17 +374,22 @@ def _verify( cls, payload: ResponsesRequest, durable_lookup: DurableBridgeLookup, + *, + payload_looks_like_full_resend: bool | None = None, ) -> "_VerifiedDurableFullResend | None": owner_account_id = durable_lookup.account_id latest_response_id = durable_lookup.latest_response_id stored_count = durable_lookup.latest_input_item_count stored_fingerprint = durable_lookup.latest_input_full_fingerprint + if owner_account_id is None or latest_response_id is None or stored_count is None or stored_fingerprint is None: + return None + full_resend_shape = ( + _http_bridge_payload_looks_like_full_resend(payload) + if payload_looks_like_full_resend is None + else payload_looks_like_full_resend + ) if ( - owner_account_id is None - or latest_response_id is None - or stored_count is None - or stored_fingerprint is None - or not _http_bridge_payload_looks_like_full_resend(payload) + not full_resend_shape or not isinstance(payload.input, list) or not _input_prefix_matches_stored_context( payload.input, @@ -441,10 +446,16 @@ def _pending_tool_calls_identity( def _verify_durable_full_resend( payload: ResponsesRequest, durable_lookup: DurableBridgeLookup | None, + *, + payload_looks_like_full_resend: bool | None = None, ) -> _VerifiedDurableFullResend | None: if durable_lookup is None or durable_lookup.account_id is None or durable_lookup.latest_response_id is None: return None - return _VerifiedDurableFullResend._verify(payload, durable_lookup) + return _VerifiedDurableFullResend._verify( + payload, + durable_lookup, + payload_looks_like_full_resend=payload_looks_like_full_resend, + ) _HTTP_BRIDGE_DEAD_OWNER_NOT_FOUND_DETAIL = "The previous bridge owner is no longer available." @@ -1210,6 +1221,11 @@ async def _stream_via_http_bridge_impl( scheduler = scheduler_for(self) clock = clock_for(self) del suppress_text_done_events + # This is a pure payload-shape signal. Capture it before the first + # await and before any legacy or durable continuity lookup so those + # paths cannot change which input shape the request presented at the + # bridge boundary. + payload_looks_like_full_resend = _http_bridge_payload_looks_like_full_resend(payload) dead_owner_anchor = False dead_owner_process_epoch_mismatch = False request_id = _denied_anchor_request_id or ensure_request_id() @@ -1472,10 +1488,13 @@ async def release_unowned_bridge_lifecycle( durable_full_resend_is_account_neutral: bool | None = None durable_full_resend_has_safe_fresh_context = False durable_full_resend_retains_required_context_cache: bool | None = None - durable_full_resend_proof = _verify_durable_full_resend(payload, durable_lookup) + durable_full_resend_proof = _verify_durable_full_resend( + payload, + durable_lookup, + payload_looks_like_full_resend=payload_looks_like_full_resend, + ) durable_full_resend_fresh_bridge_proof: _VerifiedDurableFullResend | None = None force_local_recovery_creation = False - payload_looks_like_full_resend = _http_bridge_payload_looks_like_full_resend(payload) # First-touch circuit load before any anchor planning: an expired # at-threshold poison row recorded by another replica arms this # worker's quarantine here, so the suppression checks below see it @@ -2347,8 +2366,18 @@ def switch_to_account_neutral_replay( owner_forward_fresh_replay = owner_unavailable_allows_account_neutral_replay(exc) if owner_forward_fresh_replay: switch_to_account_neutral_replay() + recovery_previous_response_id = effective_payload.previous_response_id + if ( + proxy_injected_previous_response_id + and incoming_turn_state_header is not None + and exc.failure_phase == "owner_forward" + and exc.failure_detail == "owner_input_shape_upgrade_required" + ): + # An injected anchor does not grant explicit-continuation + # recovery authority. Refresh the client's turn-state lease. + recovery_previous_response_id = None should_attempt_previous_response_recovery = not owner_forward_fresh_replay and ( - effective_payload.previous_response_id is not None + recovery_previous_response_id is not None and _http_bridge_should_attempt_local_previous_response_recovery(exc) ) should_attempt_bootstrap_rebind = ( @@ -2357,7 +2386,7 @@ def switch_to_account_neutral_replay( exc, key=bridge_session_key, headers=headers, - previous_response_id=effective_payload.previous_response_id, + previous_response_id=recovery_previous_response_id, ) ) should_attempt_turn_state_takeover = False @@ -2369,7 +2398,7 @@ def switch_to_account_neutral_replay( takeover_turn_state = _http_bridge_turn_state_anchor_for_owner_failure( exc, headers=headers, - previous_response_id=effective_payload.previous_response_id, + previous_response_id=recovery_previous_response_id, ) if takeover_turn_state is not None: # Reuse the routing lookup semantics (alias resolution @@ -2384,7 +2413,7 @@ def switch_to_account_neutral_replay( api_key_id=bridge_session_key.api_key_id, turn_state=takeover_turn_state, session_header=durable_session_header_alias, - previous_response_id=effective_payload.previous_response_id, + previous_response_id=recovery_previous_response_id, ) except Exception: logger.warning( @@ -2406,6 +2435,12 @@ def switch_to_account_neutral_replay( ) = classify_durable_full_resend(fresh_turn_state_lookup) continuity_preferred_account_id = fresh_turn_state_lookup.account_id request_state.preferred_account_id = resolve_required_account_id( + ( + "proxy-injected previous response", + request_state.preferred_account_id + if proxy_injected_previous_response_id + else None, + ), ( "refreshed previous response or bridge", continuity_preferred_account_id, diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index fa6f54fbc0..a367691c47 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -1317,6 +1317,7 @@ class _HTTPBridgeOwnerForward: owner_instance: str owner_endpoint: str key: _HTTPBridgeSessionKey + owner_process_epoch: str | None = None @dataclass(slots=True) diff --git a/app/modules/proxy/http_bridge_forwarding.py b/app/modules/proxy/http_bridge_forwarding.py index f428158615..bb8b820aec 100644 --- a/app/modules/proxy/http_bridge_forwarding.py +++ b/app/modules/proxy/http_bridge_forwarding.py @@ -22,7 +22,11 @@ from app.core.utils.request_id import get_request_id from app.core.utils.sse import format_sse_event from app.modules.api_keys.service import ApiKeyUsageReservationData -from app.modules.proxy._service.http_bridge.helpers import _http_bridge_request_budget_seconds +from app.modules.proxy._service.http_bridge.helpers import ( + _http_bridge_payload_looks_like_full_resend, + _http_bridge_request_budget_seconds, +) +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch # HTTP-only and hop-by-hop headers that must not be forwarded through the # internal bridge. These headers are either illegal in WebSocket handshakes or @@ -50,6 +54,7 @@ HTTP_BRIDGE_FORWARDED_HEADER = "x-codex-bridge-forwarded" HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER = "x-codex-bridge-origin-instance" HTTP_BRIDGE_TARGET_INSTANCE_HEADER = "x-codex-bridge-target-instance" +HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER = "x-codex-bridge-owner-process-epoch" HTTP_BRIDGE_CODEX_AFFINITY_HEADER = "x-codex-bridge-codex-session-affinity" HTTP_BRIDGE_RESERVATION_ID_HEADER = "x-codex-bridge-reservation-id" HTTP_BRIDGE_RESERVATION_KEY_ID_HEADER = "x-codex-bridge-reservation-key-id" @@ -62,18 +67,21 @@ HTTP_BRIDGE_CLIENT_IP_HEADER = "x-codex-bridge-client-ip" HTTP_BRIDGE_CLIENT_IP_SIGNATURE_HEADER = "x-codex-bridge-client-ip-signature" HTTP_BRIDGE_SIGNATURE_HEADER = "x-codex-bridge-signature" -# Additive tamper-proofing header (#1203): a second signature bound to the -# exact forwarding body (``model_dump_for_forwarding``) that is posted, so an -# in-transit rewrite injecting ``"tools": []`` is detected even though the -# primary signature hashes a plain ``model_dump`` that synthesizes the same -# empty list. Orthogonal to ``x-codex-bridge-signature-version`` below (which -# domain-separates the *primary* signature for unanchored parallel requests, -# #1169): this header carries its own full-context structured signature. Kept -# as a one-release rolling-upgrade shim alongside the legacy primary -# signature; see the ROLLOUT SHIM notes in ``build_owner_forward_headers`` and -# ``parse_forwarded_request``. +# Public full-context signature from #1203. Keep this header's codec stable: +# predecessor owners require it for file-bound forwards where primary fallback +# is forbidden. Exact input shape and owner epoch use the independent proof +# below. HTTP_BRIDGE_SIGNATURE_V2_HEADER = "x-codex-bridge-signature-v2" +HTTP_BRIDGE_TURN_STATE_SYNTHESIZED_HEADER = "x-codex-bridge-turn-state-synthesized" +HTTP_BRIDGE_TURN_STATE_PROVENANCE_SIGNATURE_HEADER = "x-codex-bridge-turn-state-provenance-signature" _HTTP_BRIDGE_SIGNATURE_VERSION_V2 = "2" +# Additive input-shape capability marker. Unlike the primary signature +# version, this header describes the classifier used for the request body. +# It is trusted only when the exact-body signature below authenticates the +# same value; an absent or unauthenticated marker keeps the legacy fallback. +HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER = "x-codex-bridge-input-shape-version" +HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER = "x-codex-bridge-input-shape-signature-v2" +_HTTP_BRIDGE_INPUT_SHAPE_VERSION_V2 = "2" @dataclass(frozen=True, slots=True) @@ -82,6 +90,7 @@ class HTTPBridgeForwardContext: target_instance: str codex_session_affinity: bool downstream_turn_state: str | None + downstream_turn_state_synthesized: bool = False original_request_unanchored: bool = False original_affinity_kind: str | None = None original_affinity_key: str | None = None @@ -89,6 +98,7 @@ class HTTPBridgeForwardContext: client_ip: str | None = None reservation: ApiKeyUsageReservationData | None = None signature_version: str | None = None + expected_owner_process_epoch: str | None = None @dataclass(frozen=True, slots=True) @@ -96,6 +106,27 @@ class HTTPBridgeForwardedRequest: context: HTTPBridgeForwardContext +@dataclass(frozen=True, slots=True) +class HTTPBridgeOwnerForwardRequest: + body: JsonObject + headers: dict[str, str] + + +def build_owner_forward_request( + *, + body: JsonObject, + headers: Mapping[str, str], + payload: ResponsesRequest, + context: HTTPBridgeForwardContext, +) -> HTTPBridgeOwnerForwardRequest: + """Build the one body/header pair authenticated on the owner wire.""" + + return HTTPBridgeOwnerForwardRequest( + body=body, + headers=build_owner_forward_headers(headers=headers, payload=payload, context=context), + ) + + @dataclass(frozen=True, slots=True) class _OwnerForwardReceiveTimeout: timeout_seconds: float @@ -146,6 +177,7 @@ def _validate_bridge_forward_context_headers(context: HTTPBridgeForwardContext) for name, value in ( (HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER, context.origin_instance), (HTTP_BRIDGE_TARGET_INSTANCE_HEADER, context.target_instance), + (HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER, context.expected_owner_process_epoch), ("x-codex-turn-state", context.downstream_turn_state), (HTTP_BRIDGE_AFFINITY_KIND_HEADER, context.original_affinity_kind), (HTTP_BRIDGE_AFFINITY_KEY_HEADER, context.original_affinity_key), @@ -172,6 +204,7 @@ async def stream_responses( headers: Mapping[str, str], context: HTTPBridgeForwardContext, request_started_at: float, + owner_supports_input_shape_classifier: bool = False, on_request_dispatched: Callable[[], None] | None = None, on_response_rejected: Callable[[], None] | None = None, on_response_wait: Callable[[], None] | None = None, @@ -179,6 +212,22 @@ async def stream_responses( scheduler: Scheduler = REAL_SCHEDULER, clock: Clock = REAL_CLOCK, ) -> AsyncIterator[str]: + if _http_bridge_owner_forward_requires_shape_upgrade(payload) and ( + not owner_supports_input_shape_classifier or not context.expected_owner_process_epoch + ): + # Classifier disagreement can either suppress required context or + # reinject a quarantined anchor. Require current-process proof + # before dispatch; existing local-recovery gates decide otherwise. + raise ProxyResponseError( + 503, + openai_error( + "bridge_owner_forward_failed", + "HTTP bridge owner cannot safely classify this continuation during a rolling upgrade", + error_type="server_error", + ), + failure_phase="owner_forward", + failure_detail="owner_input_shape_upgrade_required", + ) settings = with_dashboard_overrides(get_settings()) timeout = _owner_forward_timeout( connect_timeout_seconds=settings.upstream_connect_timeout_seconds, @@ -188,12 +237,18 @@ async def stream_responses( on_response_wait() async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: request_url = f"{owner_endpoint}{HTTP_BRIDGE_INTERNAL_FORWARD_PATH}" - request_payload = payload.model_dump_for_forwarding() - request_headers = build_owner_forward_headers(headers=headers, payload=payload, context=context) + request_payload = payload.model_dump_for_http_bridge_owner_forwarding() request_context = session.post( request_url, - json=request_payload, - headers=request_headers, + json=( + owner_request := build_owner_forward_request( + body=request_payload, + headers=headers, + payload=payload, + context=context, + ) + ).body, + headers=owner_request.headers, skip_auto_headers=_OWNER_FORWARD_SKIP_AUTO_HEADERS, ) # I/O begins when __aenter__ is awaited. Cancellation after that @@ -264,6 +319,25 @@ async def stream_responses( raise +def _http_bridge_owner_forward_requires_shape_upgrade(payload: ResponsesRequest) -> bool: + """Fence either classification reversal at a pre-change owner.""" + current_full_resend = _http_bridge_payload_looks_like_full_resend(payload) + # Pre-change owners normalize raw strings before classifying. Compare the + # normalized input's legacy item-size rule with the current raw-shape rule. + input_value = payload.input + if not isinstance(input_value, list): + return False + if len(input_value) > 1: + return not current_full_resend + if len(input_value) != 1: + return current_full_resend + try: + legacy_full_resend = len(json.dumps(input_value[0], ensure_ascii=True, separators=(",", ":"))) >= 4096 + except (OverflowError, TypeError, ValueError): + legacy_full_resend = False + return current_full_resend != legacy_full_resend + + def build_owner_forward_headers( *, headers: Mapping[str, str], @@ -310,11 +384,26 @@ def build_owner_forward_headers( forwarded[HTTP_BRIDGE_FORWARDED_HEADER] = "1" forwarded[HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER] = context.origin_instance forwarded[HTTP_BRIDGE_TARGET_INSTANCE_HEADER] = context.target_instance + if context.expected_owner_process_epoch is not None: + forwarded[HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER] = context.expected_owner_process_epoch forwarded[HTTP_BRIDGE_CODEX_AFFINITY_HEADER] = "1" if context.codex_session_affinity else "0" signature_version = _HTTP_BRIDGE_SIGNATURE_VERSION_V2 if context.original_request_unanchored else None if signature_version is not None: forwarded[HTTP_BRIDGE_SIGNATURE_VERSION_HEADER] = signature_version forwarded[HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER] = "1" + # Bind the exact posted body, shape marker, and optional owner epoch before + # additive consumers select the strongest emitted body proof. + input_shape_version = ( + None if payload._codex_lb_legacy_owner_forwarding_input_shape else _HTTP_BRIDGE_INPUT_SHAPE_VERSION_V2 + ) + if input_shape_version is not None: + forwarded[HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER] = input_shape_version + forwarded[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] = _bridge_forward_input_shape_signature( + payload=payload, + context=context, + signature_version=signature_version, + input_shape_version=input_shape_version, + ) if context.original_affinity_kind and context.original_affinity_key: forwarded[HTTP_BRIDGE_AFFINITY_KIND_HEADER] = context.original_affinity_kind forwarded[HTTP_BRIDGE_AFFINITY_KEY_HEADER] = context.original_affinity_key @@ -324,12 +413,13 @@ def build_owner_forward_headers( forwarded[HTTP_BRIDGE_FILE_OWNER_HEADER] = context.file_owner_account_id if context.client_ip: forwarded[HTTP_BRIDGE_CLIENT_IP_HEADER] = context.client_ip - forwarded[HTTP_BRIDGE_CLIENT_IP_SIGNATURE_HEADER] = _bridge_forward_signature( - payload=payload, - context=context, - include_client_ip=True, - signature_version=signature_version, - ) + if context.expected_owner_process_epoch is None: + forwarded[HTTP_BRIDGE_CLIENT_IP_SIGNATURE_HEADER] = _bridge_forward_signature( + payload=payload, + context=context, + include_client_ip=True, + signature_version=signature_version, + ) if context.downstream_turn_state: forwarded["x-codex-turn-state"] = context.downstream_turn_state if context.reservation is not None: @@ -343,22 +433,27 @@ def build_owner_forward_headers( # updated origins during a rolling upgrade. New-code receivers verify the # tamper-proofing header below first and fall back to this primary # signature only when the tamper-proofing header does not validate. - forwarded[HTTP_BRIDGE_SIGNATURE_HEADER] = _bridge_forward_signature( - payload=payload, - context=context, - include_client_ip=False, - signature_version=signature_version, - ) - # Additive tamper-proofing signature bound to the exact posted forwarding - # body; covers the full authenticated context (including the unanchored / - # signature-version domain) so it cannot be replayed against a different - # forward. - forwarded[HTTP_BRIDGE_SIGNATURE_V2_HEADER] = _bridge_forward_tools_bound_signature( - payload=payload, - context=context, - signature_version=signature_version, - ) - return forwarded + # Capability-gated requests must also reject a predecessor process that + # ignores the signed epoch after a rollback. Do not give it a valid + # primary fallback. + if context.expected_owner_process_epoch is None: + forwarded[HTTP_BRIDGE_SIGNATURE_HEADER] = _bridge_forward_signature( + payload=payload, + context=context, + include_client_ip=False, + signature_version=signature_version, + ) + # Keep the public pre-input-shape full-context proof byte-compatible. It is + # the only file-owner proof understood by a predecessor owner, and primary + # fallback is deliberately forbidden for file-bearing forwards. An + # epoch-gated request must not carry any proof that a predecessor accepts. + if context.expected_owner_process_epoch is None: + forwarded[HTTP_BRIDGE_SIGNATURE_V2_HEADER] = _bridge_forward_tools_bound_signature( + payload=payload, + context=context, + signature_version=signature_version, + ) + return _with_bridge_turn_state_provenance(forwarded, context=context) def parse_forwarded_request( @@ -388,7 +483,13 @@ def parse_forwarded_request( ) client_ip = _optional_header(headers.get(HTTP_BRIDGE_CLIENT_IP_HEADER)) signature_version = _optional_header(headers.get(HTTP_BRIDGE_SIGNATURE_VERSION_HEADER)) + input_shape_version = _optional_header(headers.get(HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER)) + if input_shape_version not in {None, _HTTP_BRIDGE_INPUT_SHAPE_VERSION_V2}: + return None, _invalid_bridge_forward_signature_error() original_unanchored_value = _optional_header(headers.get(HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER)) + turn_state_synthesized_value = _optional_header(headers.get(HTTP_BRIDGE_TURN_STATE_SYNTHESIZED_HEADER)) + if turn_state_synthesized_value not in {None, "0", "1"}: + return None, _invalid_bridge_forward_signature_error() if signature_version == _HTTP_BRIDGE_SIGNATURE_VERSION_V2: if original_unanchored_value not in {"0", "1"}: return None, _invalid_bridge_forward_signature_error() @@ -402,6 +503,7 @@ def parse_forwarded_request( target_instance=target_instance, codex_session_affinity=_bool_header(headers.get(HTTP_BRIDGE_CODEX_AFFINITY_HEADER)), downstream_turn_state=_optional_header(headers.get("x-codex-turn-state")), + downstream_turn_state_synthesized=turn_state_synthesized_value == "1", original_request_unanchored=original_request_unanchored, original_affinity_kind=_optional_header(headers.get(HTTP_BRIDGE_AFFINITY_KIND_HEADER)), original_affinity_key=_optional_header(headers.get(HTTP_BRIDGE_AFFINITY_KEY_HEADER)), @@ -409,9 +511,10 @@ def parse_forwarded_request( client_ip=client_ip, reservation=_reservation_from_headers(headers), signature_version=signature_version, + expected_owner_process_epoch=_optional_header(headers.get(HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER)), ) - # Tamper-proofing fast path (#1203): a VALIDATING tamper-proofing - # signature proves the received body was not rewritten in transit — + # Exact-shape fast path: a validating shape signature proves the received + # body was not rewritten in transit — # including an injected ``"tools": []`` that the primary plain-dump # signature cannot distinguish from an omitted field — and it also # authenticates the full forward context (structured, delimiter-safe), so @@ -421,6 +524,8 @@ def parse_forwarded_request( # value on an honestly primary-signed forward, so a present-but-invalid # header simply falls through to the primary verification. tools_bound_signature = _optional_header(headers.get(HTTP_BRIDGE_SIGNATURE_V2_HEADER)) + authenticated_body_signatures: list[str] = [] + forward_authenticated = False tools_bound_valid = tools_bound_signature is not None and hmac.compare_digest( tools_bound_signature, _bridge_forward_tools_bound_signature( @@ -429,8 +534,59 @@ def parse_forwarded_request( signature_version=signature_version, ), ) + input_shape_signature = _optional_header(headers.get(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER)) + expected_input_shape_signature = _bridge_forward_input_shape_signature( + payload=payload, + context=context, + signature_version=signature_version, + input_shape_version=input_shape_version, + ) + input_shape_valid = input_shape_signature is not None and hmac.compare_digest( + input_shape_signature, expected_input_shape_signature + ) + # The currently deployed shape-v2 build used the old V2 header for these + # exact same bytes. Accept that origin layout during the controlled cutover. + deployed_shape_v2_valid = tools_bound_signature is not None and hmac.compare_digest( + tools_bound_signature, expected_input_shape_signature + ) + if input_shape_valid or deployed_shape_v2_valid: + authenticated_body_signature = input_shape_signature if input_shape_valid else tools_bound_signature + assert authenticated_body_signature is not None + authenticated_body_signatures.append(authenticated_body_signature) + forward_authenticated = True + if ( + context.expected_owner_process_epoch is not None + and context.expected_owner_process_epoch != http_bridge_owner_process_epoch() + ): + return None, ProxyResponseError( + 503, + openai_error( + "bridge_owner_forward_failed", + "Internal bridge forward reached a different owner process", + error_type="server_error", + ), + ) + payload._codex_lb_input_shape_wire_version = input_shape_version + payload._codex_lb_legacy_owner_forwarding_input_shape = ( + input_shape_version != _HTTP_BRIDGE_INPUT_SHAPE_VERSION_V2 + ) + # Neither predecessor codec authenticates process epoch. Refuse the claim + # before considering their proofs unless the exact-shape proof validated. + if context.expected_owner_process_epoch is not None and not forward_authenticated: + return None, _invalid_bridge_forward_signature_error() + if tools_bound_valid and not input_shape_valid and not deployed_shape_v2_valid: + payload._codex_lb_input_shape_wire_version = None + payload._codex_lb_legacy_owner_forwarding_input_shape = True if tools_bound_valid: - return HTTPBridgeForwardedRequest(context=context), None + assert tools_bound_signature is not None + authenticated_body_signatures.append(tools_bound_signature) + forward_authenticated = True + if forward_authenticated or context.downstream_turn_state_synthesized: + return _authenticated_bridge_forward_result( + headers=headers, + context=context, + authenticated_body_signatures=authenticated_body_signatures, + ) if context.file_owner_account_id is not None or extract_input_file_ids(payload.input): # The rolling-upgrade primary signature does not bind the additive # file-owner proof. Never allow a stripped/forged proof to downgrade to @@ -476,6 +632,19 @@ def parse_forwarded_request( ) if not signature_valid: return None, _invalid_bridge_forward_signature_error() + # A pre-change origin posts the normalized one-item array for a client + # string and has no body-bound signature that can prove the original wire + # shape. Keep that array out of the size-based full-resend heuristic on + # the new owner: treating its JSON overhead as client content can cross + # the 4096-byte boundary and drop otherwise-valid prior context during a + # rolling upgrade. Current origins send the body-bound signature and + # preserve raw strings on the owner wire, so their classification remains + # exact. + # The additive marker is only authenticated by the exact-body signature. + # A primary-signature fallback therefore remains legacy even when an + # unauthenticated ``...-input-shape-version: 2`` header was supplied. + payload._codex_lb_input_shape_wire_version = None + payload._codex_lb_legacy_owner_forwarding_input_shape = True return HTTPBridgeForwardedRequest(context=context), None @@ -607,19 +776,11 @@ def _bridge_forward_tools_bound_signature( context: HTTPBridgeForwardContext, signature_version: str | None = None, ) -> str: - """Tamper-proofing signature bound to the exact posted forwarding body. - - Signs the same forwarding dump that is actually posted - (``model_dump_for_forwarding``), not a plain ``model_dump`` that - synthesizes ``"tools": []`` for clients that omitted the field. A plain - dump would make the omitted-tools and explicit-``tools: []`` bodies sign - identically, so a body rewritten in transit to inject ``"tools": []`` - would still verify on the owner instance and re-mark ``tools`` as - explicitly set (issue #1184). Uses a distinct protocol domain so it can - never be confused with the primary signature, and reuses the same - canonical structured encoding (covering the full authenticated context, - including the unanchored / signature-version domain) so the binding also - carries #1169's isolation guarantees. Always authenticates ``client_ip``. + """Public pre-input-shape full-context signature. + + Keep this codec byte-compatible for predecessor file-owner forwards. The + independent exact-shape proof below owns the posted-body, marker, and epoch + binding added by this change. """ body_digest = _bridge_forward_body_digest(payload.model_dump_for_forwarding()) signing_payload = _structured_bridge_signing_payload( @@ -632,11 +793,109 @@ def _bridge_forward_tools_bound_signature( return _sign_bridge_payload(signing_payload) +def _bridge_forward_input_shape_signature( + *, + payload: ResponsesRequest, + context: HTTPBridgeForwardContext, + signature_version: str | None = None, + input_shape_version: str | None = _HTTP_BRIDGE_INPUT_SHAPE_VERSION_V2, +) -> str: + """Authenticate the exact owner body, input shape, and optional epoch.""" + + body_digest = _bridge_forward_body_digest(payload.model_dump_for_http_bridge_owner_forwarding()) + signing_payload = _structured_bridge_signing_payload( + body_digest=body_digest, + context=context, + include_client_ip=True, + signature_version=signature_version, + protocol="codex-lb-http-bridge-forward-tools-bound", + input_shape_version=input_shape_version, + include_owner_process_epoch=True, + ) + return _sign_bridge_payload(signing_payload) + + def _bridge_forward_body_digest(payload_dump: JsonObject) -> str: payload_json = json.dumps(payload_dump, ensure_ascii=True, sort_keys=True, separators=(",", ":")) return hashlib.sha256(payload_json.encode("utf-8")).hexdigest() +def _with_bridge_turn_state_provenance( + forwarded: dict[str, str], + *, + context: HTTPBridgeForwardContext, +) -> dict[str, str]: + """Bind synthesized-state privilege to the strongest emitted body proof.""" + + if not context.downstream_turn_state_synthesized: + return forwarded + authenticated_body_signature = _bridge_forward_provenance_body_signature(forwarded) + assert authenticated_body_signature is not None + forwarded[HTTP_BRIDGE_TURN_STATE_SYNTHESIZED_HEADER] = "1" + forwarded[HTTP_BRIDGE_TURN_STATE_PROVENANCE_SIGNATURE_HEADER] = _bridge_forward_turn_state_provenance_signature( + context=context, + authenticated_body_signature=authenticated_body_signature, + ) + return forwarded + + +def _authenticated_bridge_forward_result( + *, + headers: Mapping[str, str], + context: HTTPBridgeForwardContext, + authenticated_body_signatures: list[str], +) -> tuple[HTTPBridgeForwardedRequest | None, ProxyResponseError | None]: + """Grant synthesized-state privilege only for the authenticated body proof.""" + + if context.downstream_turn_state_synthesized: + provenance_signature = _optional_header(headers.get(HTTP_BRIDGE_TURN_STATE_PROVENANCE_SIGNATURE_HEADER)) + provenance_valid = provenance_signature is not None and any( + hmac.compare_digest( + provenance_signature, + _bridge_forward_turn_state_provenance_signature( + context=context, + authenticated_body_signature=authenticated_body_signature, + ), + ) + for authenticated_body_signature in authenticated_body_signatures + ) + if not provenance_valid: + return None, _invalid_bridge_forward_signature_error() + return HTTPBridgeForwardedRequest(context=context), None + + +def _bridge_forward_turn_state_provenance_signature( + *, + context: HTTPBridgeForwardContext, + authenticated_body_signature: str, +) -> str: + """Bind generated-state privilege to body-proof bytes that already validate.""" + + signing_payload = json.dumps( + { + "downstream_turn_state_synthesized": context.downstream_turn_state_synthesized, + "protocol": "codex-lb-http-bridge-turn-state-provenance-v1", + # Keep the field name and wire bytes stable for the pre-shape V2 + # proof. The value may also be the independently validated exact- + # shape proof when that additive contract is present. + "tools_bound_signature": authenticated_body_signature, + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + return _sign_bridge_payload(signing_payload) + + +def _bridge_forward_provenance_body_signature(headers: Mapping[str, str]) -> str | None: + """Select the strongest emitted body proof for synthesized provenance.""" + + exact_shape_signature = _optional_header(headers.get("x-codex-bridge-input-shape-signature-v2")) + if exact_shape_signature is not None: + return exact_shape_signature + return _optional_header(headers.get(HTTP_BRIDGE_SIGNATURE_V2_HEADER)) + + def _structured_bridge_signing_payload( *, body_digest: str, @@ -644,36 +903,43 @@ def _structured_bridge_signing_payload( include_client_ip: bool, signature_version: str | None, protocol: str, + input_shape_version: str | None = None, + include_owner_process_epoch: bool = False, ) -> str: # Canonical structured encoding: object boundaries make field re-packing # impossible, the client-IP mode is itself authenticated, and ``protocol`` # domain-separates the primary and tamper-proofing signatures. + signing_fields = { + "body_digest": body_digest, + "client_ip": context.client_ip if include_client_ip else None, + "client_ip_present": context.client_ip is not None, + "codex_session_affinity": context.codex_session_affinity, + "downstream_turn_state": context.downstream_turn_state, + "file_owner_account_id": context.file_owner_account_id, + "include_client_ip": include_client_ip, + "origin_instance": context.origin_instance, + "original_affinity_key": context.original_affinity_key, + "original_affinity_kind": context.original_affinity_kind, + "original_request_unanchored": context.original_request_unanchored, + "protocol": protocol, + "reservation": ( + { + "id": context.reservation.reservation_id, + "key_id": context.reservation.key_id, + "model": context.reservation.model, + } + if context.reservation is not None + else None + ), + "signature_version": signature_version, + "target_instance": context.target_instance, + } + if input_shape_version is not None: + signing_fields["input_shape_version"] = input_shape_version + if include_owner_process_epoch and context.expected_owner_process_epoch is not None: + signing_fields["expected_owner_process_epoch"] = context.expected_owner_process_epoch return json.dumps( - { - "body_digest": body_digest, - "client_ip": context.client_ip if include_client_ip else None, - "client_ip_present": context.client_ip is not None, - "codex_session_affinity": context.codex_session_affinity, - "downstream_turn_state": context.downstream_turn_state, - "file_owner_account_id": context.file_owner_account_id, - "include_client_ip": include_client_ip, - "origin_instance": context.origin_instance, - "original_affinity_key": context.original_affinity_key, - "original_affinity_kind": context.original_affinity_kind, - "original_request_unanchored": context.original_request_unanchored, - "protocol": protocol, - "reservation": ( - { - "id": context.reservation.reservation_id, - "key_id": context.reservation.key_id, - "model": context.reservation.model, - } - if context.reservation is not None - else None - ), - "signature_version": signature_version, - "target_instance": context.target_instance, - }, + signing_fields, ensure_ascii=True, sort_keys=True, separators=(",", ":"), diff --git a/app/modules/proxy/ring_membership.py b/app/modules/proxy/ring_membership.py index 99ded475e0..7c5841c0e0 100644 --- a/app/modules/proxy/ring_membership.py +++ b/app/modules/proxy/ring_membership.py @@ -4,6 +4,7 @@ import uuid from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from dataclasses import dataclass from datetime import datetime from hashlib import sha256 from typing import TYPE_CHECKING, Any, cast @@ -16,6 +17,7 @@ from app.core.utils.time import utcnow from app.db.models import BridgeRingMember from app.db.session import close_session +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch if TYPE_CHECKING: from collections.abc import Callable @@ -25,6 +27,15 @@ RING_STALE_THRESHOLD_SECONDS = 30 RING_STALE_GRACE_SECONDS = RING_HEARTBEAT_INTERVAL_SECONDS + 5 RING_MEMBER_RETENTION_SECONDS = 24 * 60 * 60 +HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY = "responses-input-shape-classifier-v2" +_CURRENT_HTTP_BRIDGE_RING_CAPABILITIES = (HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY,) + + +@dataclass(frozen=True, slots=True) +class _BridgeRingAdvertisement: + endpoint_base_url: str | None + owner_process_epoch: str | None + capabilities: frozenset[str] class RingMembershipService: @@ -214,6 +225,36 @@ async def resolve_endpoint( metadata_json = result.scalar_one_or_none() return _bridge_ring_endpoint_from_metadata(metadata_json) + async def owner_supports_capability( + self, + instance_id: str, + *, + owner_process_epoch: str, + capability: str, + stale_threshold_seconds: int = RING_STALE_THRESHOLD_SECONDS, + ) -> bool: + """Return capability proof for the exact live owner process.""" + from datetime import timedelta + + cutoff = utcnow() - timedelta(seconds=stale_threshold_seconds) + async with self._session() as session: + result = await session.execute( + select(BridgeRingMember.metadata_json) + .where( + BridgeRingMember.instance_id == instance_id, + BridgeRingMember.last_heartbeat_at >= cutoff, + ) + .limit(1) + ) + metadata_json = result.scalar_one_or_none() + advertisement = _bridge_ring_advertisement_from_metadata(metadata_json) + return bool( + advertisement is not None + and advertisement.endpoint_base_url is not None + and advertisement.owner_process_epoch == owner_process_epoch + and capability in advertisement.capabilities + ) + async def ring_fingerprint(self, stale_threshold_seconds: int = RING_STALE_THRESHOLD_SECONDS) -> str: """sha256 of sorted active member list. Same for all pods with same membership.""" members = await self.list_active(stale_threshold_seconds) @@ -232,10 +273,23 @@ async def _session(self) -> AsyncIterator[AsyncSession]: def _bridge_ring_metadata_json(endpoint_base_url: str | None) -> str | None: if endpoint_base_url is None: return None - return json.dumps({"endpoint_base_url": endpoint_base_url}, ensure_ascii=True, separators=(",", ":")) + return json.dumps( + { + "endpoint_base_url": endpoint_base_url, + "owner_process_epoch": http_bridge_owner_process_epoch(), + "capabilities": list(_CURRENT_HTTP_BRIDGE_RING_CAPABILITIES), + }, + ensure_ascii=True, + separators=(",", ":"), + ) def _bridge_ring_endpoint_from_metadata(metadata_json: str | None) -> str | None: + advertisement = _bridge_ring_advertisement_from_metadata(metadata_json) + return advertisement.endpoint_base_url if advertisement is not None else None + + +def _bridge_ring_advertisement_from_metadata(metadata_json: str | None) -> _BridgeRingAdvertisement | None: if metadata_json is None: return None try: @@ -245,7 +299,18 @@ def _bridge_ring_endpoint_from_metadata(metadata_json: str | None) -> str | None if not isinstance(payload, dict): return None endpoint = payload.get("endpoint_base_url") - if not isinstance(endpoint, str): - return None - stripped = endpoint.strip().rstrip("/") - return stripped or None + stripped_endpoint = endpoint.strip().rstrip("/") if isinstance(endpoint, str) else "" + owner_process_epoch = payload.get("owner_process_epoch") + if not isinstance(owner_process_epoch, str) or not owner_process_epoch: + owner_process_epoch = None + raw_capabilities = payload.get("capabilities") + capabilities = ( + frozenset(raw_capabilities) + if isinstance(raw_capabilities, list) and all(isinstance(item, str) for item in raw_capabilities) + else frozenset() + ) + return _BridgeRingAdvertisement( + endpoint_base_url=stripped_endpoint or None, + owner_process_epoch=owner_process_epoch, + capabilities=capabilities, + ) diff --git a/openspec/changes/preserve-http-bridge-input-shape/.openspec.yaml b/openspec/changes/preserve-http-bridge-input-shape/.openspec.yaml new file mode 100644 index 0000000000..1ea7e36f49 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-input-shape/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-09 diff --git a/openspec/changes/preserve-http-bridge-input-shape/context.md b/openspec/changes/preserve-http-bridge-input-shape/context.md new file mode 100644 index 0000000000..dd3aa4b180 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-input-shape/context.md @@ -0,0 +1,34 @@ +# Context + +This is the request compatibility slice of #1953 on current main. It preserves the accepted candidate implementation without requiring the quarantine-generation sibling. A short raw string must remain short after validation; two function-call outputs need the anchor holding their calls. + +Mixed-version forwarding carries authenticated input-shape metadata, checks the advertised owner process, and uses existing local recovery for eligible pre-dispatch rejections. Whole-scope owner approval remains required. These policies are extracted, not newly approved here. + +Current-head review reproduced a missing anchor for one 4096-character function-call output on the public Responses route. Tool-output-only classification therefore covers nonempty arrays of any size. Legacy receiver semantics remain unchanged; the existing classification-disagreement gate now fences this case too. + +## Signing compatibility and rollout + +PR 2088 provenance is additive: it signs the authenticated body proof selected +by the forwarding path and does not require changing either known +pre-provenance codec. The two codecs are the public pre-input-shape V2 envelope +and the shape-V2 envelope already deployed at `9ede3db64fbb71300aa934b767a4f4db79d4a5b4`. +Treating only the first as “the deployed codec” caused the original composite +report to conflate PR 2088 provenance with PR 2277's older rolling gap. + +The repaired wire keeps the public V2 envelope in +`x-codex-bridge-signature-v2` and moves the byte-identical deployed shape-V2 +envelope to `x-codex-bridge-input-shape-signature-v2`. A new receiver accepts +old `9ede3db` origins that still carry those shape-V2 bytes in the old header. +The reverse direction is not symmetric for file-bound or epoch-gated requests: +an old `9ede3db` owner does not know the new header, and primary fallback is +correctly unavailable. Therefore `9ede3db -> repaired build` requires an +all-owner stop/start cutover. The current SQLite deployment has ring size one, +so that boundary is operationally available; it is not a supported mixed-owner +rolling boundary. Public pre-input-shape owners remain compatible with ordinary +new forwards, including file-bound forwards, through the preserved V2 proof. + +For example, an ordinary file forward carries public V2 plus exact-shape V2. +A public predecessor verifies the first; a repaired owner verifies the second. +An epoch-gated ambiguous continuation carries only exact-shape V2, so any owner +that cannot authenticate the marker and epoch fails closed before continuity +selection. diff --git a/openspec/changes/preserve-http-bridge-input-shape/proposal.md b/openspec/changes/preserve-http-bridge-input-shape/proposal.md new file mode 100644 index 0000000000..a05cfa51eb --- /dev/null +++ b/openspec/changes/preserve-http-bridge-input-shape/proposal.md @@ -0,0 +1,23 @@ +## Why + +Responses validation and internal forwarding can change full-resend classification and discard an anchor required by parallel tool outputs. This extracts the request compatibility concern from #1953 and addresses #2269 independently of quarantine generation changes. + +## What Changes + +- Preserve raw input provenance through validation and authenticated owner forwarding. +- Keep output-only tool continuations anchored. +- Bind mixed-version capability checks to owner process identity and preserve existing safe local recovery. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: input-shape classification and owner-forward compatibility. + +## Impact + +Request normalization, HTTP bridge selection and owner forwarding, bridge ring metadata, and regressions. No new settings or schema. Quarantine lifetime is unchanged. diff --git a/openspec/changes/preserve-http-bridge-input-shape/specs/responses-api-compat/spec.md b/openspec/changes/preserve-http-bridge-input-shape/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..50afebbe6d --- /dev/null +++ b/openspec/changes/preserve-http-bridge-input-shape/specs/responses-api-compat/spec.md @@ -0,0 +1,222 @@ +## ADDED Requirements + +### Requirement: Quarantine selection distinguishes local reuse from durable context + +An active quarantine MUST make every live session under its key unavailable for +local session reuse and MUST make that live session count as absent when +determining whether a local bridge can supply an anchor. A full-conversation +resend MAY therefore suppress proxy anchor injection and proceed with its own +untrimmed input. A genuine delta-only continuation MUST retain access to its +durable anchor, because quarantine does not erase durable context and the +request has no equivalent replacement context source. This distinction MUST +not mutate account health, routing, or durable ownership. + +For this requirement, the canonical full-resend-shape predicate MUST inspect +the decoded Responses request's `input` before durable lookup or replay +projection. It MUST classify each shape as follows: + +- A string is full-resend-shaped if it contains at least 4096 Unicode code + points; a shorter string is delta-only. +- An array with more than one item is full-resend-shaped unless every item is + a `function_call_output`, `custom_tool_call_output`, or + `apply_patch_call_output`. An array containing only those output items is + delta-only because their corresponding calls exist behind the anchor. +- A one-item array containing one of those tool-output types is delta-only, + regardless of output length. Other one-item arrays are full-resend-shaped if + the compact serialization of the + entire array (`ensure_ascii=true` and no separator whitespace) contains at + least 4096 characters. A shorter serialization or a serialization failure + is delta-only. +- Empty arrays, null input, and all other shapes are delta-only. + +Every length threshold in the current and legacy predicates MUST count +characters, not UTF-8 bytes. Raw text uses Unicode code points; compact JSON +uses the characters in its ASCII-escaped serialization, including escapes and +punctuation. Exactly 4096 is included and 4095 is not. This is only a +payload-shape signal and does not establish durable full-resend proof, +prefix identity, or account-neutral replay safety. Request validation MUST +preserve a client-supplied string's original shape and character length for +this decision; normalizing that string into a one-item array MUST NOT add the +array envelope to its boundary calculation. An internal HTTP bridge +owner-forward hop MUST preserve that original string shape so the owner's +request validation reaches the same classification as the origin. +During a rolling upgrade, when an older origin forwards only a normalized +one-item array and the owner cannot validate the additive exact-body signature, +the owner MUST use conservative canonical-shape precedence. An exact canonical +normalized raw-string shape (`role=user` with one `input_text` part) MUST be +classified by its contained text length, not by its array or item serialization. +That wire shape is byte-identical to a genuine client array of the same form, so +the owner cannot recover the original provenance; the contained-text rule +therefore applies to both origins. A noncanonical one-item array MUST retain the +legacy compact-item predicate. Neither path may count a normalization envelope +as client text. In the +inverse rolling-upgrade direction, an upgraded origin without positive proof +that the selected owner implements this classifier MUST NOT dispatch an input +whose current and legacy classifications disagree in either direction. This +guard MUST cover a client string below 4096 characters whose normalized item +reaches 4096 compact-serialization characters, a multi-item array containing +only the allowed tool-output item types, a single tool output whose legacy +item serialization reaches 4096 characters, and a one-item array whose whole-array +serialization reaches 4096 characters while its item serialization does not. +It MUST also cover full-resend-shaped system/developer-only arrays that +normalize to empty input. Truly empty input and small single-message arrays +whose current and legacy classifications are both delta-only MUST remain +outside this upgrade requirement. +The origin MUST fail +closed or enter an already-authorized local recovery path before owner I/O. +For a request with turn state and no client `previous_response_id`, the typed +`owner_forward` / `owner_input_shape_upgrade_required` failure MUST be eligible +for the existing local turn-state takeover path. Takeover MUST still require a +successful fresh durable lookup with no active owner lease and the existing +continuity-routing checks. A failed lookup or active lease MUST fail closed; +missing owner capability proof MUST NOT permit owner dispatch. +A proxy-injected durable anchor MUST NOT count as a client-supplied previous +response id for this decision. The origin MUST resolve fresh turn-state +ownership without using that injected anchor as a lookup alias, and any retained +anchor MUST remain constrained to its original account. +Positive proof MUST come from a live bridge-ring advertisement containing the +exact input-shape-classifier capability and a process epoch equal to the +durable owner's recorded `owner_process_epoch`. When that proof matches, the +origin MAY dispatch the classification-ambiguous shape to the upgraded owner. +Missing, malformed, stale, or epoch-mismatched advertisements MUST NOT +authorize dispatch, including an advertisement left by an earlier process +that reused the same instance id. + +Capability-gated forwards MUST carry `x-codex-bridge-owner-process-epoch` +with the proven process epoch, authenticated by the exact-body signature. +The receiving owner MUST reject a signed epoch unequal to its local process +epoch before continuity selection. A nonempty process epoch MUST NOT be +authorized by either the legacy primary proof or the public pre-input-shape V2 +proof, because neither codec authenticates that field. These forwards MUST carry only +`x-codex-bridge-input-shape-signature-v2` as their body proof and MUST NOT +include either the legacy primary signature or the public +`x-codex-bridge-signature-v2` proof. A predecessor process that ignores the +epoch therefore cannot accept the request after a rollback. + +An upgraded owner-forward request MUST advertise +`x-codex-bridge-input-shape-version: 2` when it posts a body whose exact input +shape is known. The exact posted body, marker value, and optional owner process +epoch MUST be authenticated by the independent +`x-codex-bridge-input-shape-signature-v2` proof. +The owner MUST trust the current-shape mode only when that signature validates +with the same header value; a missing, malformed, or primary-signature-only +marker MUST keep legacy compatibility classification. Receivers implementing +this versioned input-shape contract MUST reject an unsupported nonempty +version before the forwarded request reaches continuity selection. + +For ordinary forwards, `x-codex-bridge-signature-v2` MUST retain the public +pre-input-shape codec: it signs `model_dump_for_forwarding()` and MUST NOT add +the input-shape marker or owner process epoch to its structured payload. The +origin MUST send this proof alongside the independent exact-shape proof so a +pre-input-shape owner can authenticate file-bound forwards without using the +legacy primary fallback. An upgraded receiver MUST also accept the previously +deployed shape-v2 layout in which the exact-shape proof bytes were carried in +`x-codex-bridge-signature-v2`; this acceptance MUST still require the signed +marker and, when present, the matching local owner process epoch. + +The marker advertises the origin's input-shape provenance, not the selected +owner's capability. For inputs whose current and legacy classifications agree, +forwarding MUST NOT require classifier capability proof and MUST retain the +existing primary-signature fallback, subject to its existing restrictions. +A predecessor owner may ignore the additive marker and exact-shape proof and +validate the public full-context proof, including for file-bound requests. +The origin MUST NOT remove a known-shape marker merely because the destination +lacks classifier capability proof. + +A payload already classified under legacy forwarding MUST omit the marker +when forwarded again. Its exact-body signature MUST bind the posted body +without an input-shape version. A current receiver accepting that signature +MUST retain legacy compatibility classification. This MUST NOT bypass the +capability and process-epoch checks for classification-ambiguous requests. + +#### Scenario: Quarantine preserves durable context for delta-only requests + +- **GIVEN** a live bridge session is quarantined and its durable anchor is + available +- **WHEN** a genuine delta-only continuation arrives for that session key +- **THEN** the quarantined live session is excluded from local reuse and + full-resend anchor injection +- **AND** the request still resolves and receives its durable anchor +- **AND** no account health, routing, or durable ownership state changes + +#### Scenario: Legacy owner forwarding uses canonical normalized text length + +- **GIVEN** an older origin normalized a below-boundary client string into a + one-item array before forwarding it to a newer owner +- **AND** the forward validates only through the rolling-upgrade legacy + signature fallback +- **WHEN** the newer owner classifies the request shape +- **THEN** it MUST classify the exact canonical normalized shape by its + contained text length, not by its normalization envelope +- **AND** it MUST retain the durable previous-response anchor + +#### Scenario: Legacy fallback uses the compact-item predicate for noncanonical arrays + +- **GIVEN** an older origin forwards a genuinely noncanonical one-item array + such as `["x" * 4094]` +- **AND** the forward validates only through the rolling-upgrade legacy + signature fallback +- **WHEN** the newer owner classifies the request shape +- **THEN** it MUST use the compact serialization of that item for the 4096-character + boundary +- **AND** it MUST classify the item as full-resend-shaped at exactly 4096 + characters + +#### Scenario: Unauthenticated input-shape marker stays legacy + +- **GIVEN** a forwarded body carries `x-codex-bridge-input-shape-version: 2` +- **AND** its exact-body signature is missing or does not bind that marker +- **WHEN** the owner validates the primary bridge signature +- **THEN** it MUST accept only under legacy compatibility classification +- **AND** it MUST NOT infer current-shape mode from the marker alone + +#### Scenario: Owner is replaced after capability proof + +- **GIVEN** an origin proved the selected owner's process epoch and classifier + capability for an ambiguous delta-only request +- **WHEN** a replacement process receives the forward at the same instance id +- **THEN** an upgraded replacement MUST reject the signed process-epoch mismatch +- **AND** a predecessor replacement MUST fail signature validation without a + legacy primary fallback +- **AND** neither replacement may select continuity or suppress the durable anchor + +#### Scenario: Current origin does not expose a delta to a legacy owner + +- **GIVEN** an upgraded origin selects a remote owner whose current classifier + capability is not positively known +- **AND** the request is delta-only under the current classifier but full-resend + shaped after legacy normalization +- **WHEN** the origin reaches the owner-forward boundary +- **THEN** it MUST NOT dispatch the request to that owner +- **AND** it MUST fail closed or use an already-authorized local recovery path + before the legacy owner can suppress the durable anchor + +#### Scenario: Proven upgraded owner receives an ambiguous delta + +- **GIVEN** an upgraded origin selects a live remote owner +- **AND** the ring advertises the exact input-shape-classifier capability with + a process epoch equal to the durable owner's recorded process epoch +- **AND** the request is delta-only under the current classifier but + full-resend shaped after legacy normalization +- **WHEN** the origin reaches the owner-forward boundary +- **THEN** it MAY dispatch the request to that owner +- **AND** the owner MUST retain the durable previous-response anchor + +#### Scenario: Replaced owner process cannot inherit capability proof + +- **GIVEN** an instance id has a classifier-capable ring advertisement from an + earlier owner process +- **AND** the durable owner record names a different current process epoch +- **WHEN** an upgraded origin evaluates an ambiguous delta-only owner forward +- **THEN** the stale advertisement MUST NOT authorize dispatch +- **AND** the origin MUST fail closed or use an already-authorized local + recovery path before owner I/O + +#### Scenario: Large single tool output retains its continuation anchor + +- **GIVEN** a quarantined session has a durable turn-state anchor +- **AND** a request contains one tool output whose serialized size exceeds 4096 characters +- **AND** the client omits `previous_response_id` +- **WHEN** the bridge classifies the request +- **THEN** it MUST treat the input as delta-only and retain the durable anchor +- **AND** forwarding to an owner with a disagreeing legacy classifier MUST require the existing capability and process-epoch proof diff --git a/openspec/changes/preserve-http-bridge-input-shape/tasks.md b/openspec/changes/preserve-http-bridge-input-shape/tasks.md new file mode 100644 index 0000000000..b8426b9962 --- /dev/null +++ b/openspec/changes/preserve-http-bridge-input-shape/tasks.md @@ -0,0 +1,11 @@ +## Implementation + +- [x] Extract raw input preservation, classifier, forwarding authentication, capability proof and local recovery. +- [x] Preserve target quarantine implementation and main-native APIs. +- [x] Verify independent route and forwarding behavior. +- [x] Validate lint, typing and OpenSpec. +- [x] Verify both merge orders with independent quarantine slice. + +## Owner acceptance + +- [ ] Obtain whole-scope owner approval of forwarding compatibility and capability-gate behavior before merge or archival. diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 9b0b8d4d2b..85b660f989 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -2375,6 +2375,227 @@ When an upstream websocket or HTTP bridge session has multiple pending Responses - **AND** only the expired oldest request is failed - **AND** the younger request remains pending +### Requirement: Quarantine selection distinguishes local reuse from durable context + +An active quarantine MUST make every live session under its key unavailable for +local session reuse and MUST make that live session count as absent when +determining whether a local bridge can supply an anchor. A full-conversation +resend MAY therefore suppress proxy anchor injection and proceed with its own +untrimmed input. A genuine delta-only continuation MUST retain access to its +durable anchor, because quarantine does not erase durable context and the +request has no equivalent replacement context source. This distinction MUST +not mutate account health, routing, or durable ownership. + +For this requirement, the canonical full-resend-shape predicate MUST inspect +the decoded Responses request's `input` before durable lookup or replay +projection. It MUST classify each shape as follows: + +- A string is full-resend-shaped if it contains at least 4096 Unicode code + points; a shorter string is delta-only. +- An array with more than one item is full-resend-shaped unless every item is + a `function_call_output`, `custom_tool_call_output`, or + `apply_patch_call_output`. An array containing only those output items is + delta-only because their corresponding calls exist behind the anchor. +- A one-item array containing one of those tool-output types is delta-only, + regardless of output length. Other one-item arrays are full-resend-shaped if + the compact serialization of the + entire array (`ensure_ascii=true` and no separator whitespace) contains at + least 4096 characters. A shorter serialization or a serialization failure + is delta-only. +- Empty arrays, null input, and all other shapes are delta-only. + +Every length threshold in the current and legacy predicates MUST count +characters, not UTF-8 bytes. Raw text uses Unicode code points; compact JSON +uses the characters in its ASCII-escaped serialization, including escapes and +punctuation. Exactly 4096 is included and 4095 is not. This is only a +payload-shape signal and does not establish durable full-resend proof, +prefix identity, or account-neutral replay safety. Request validation MUST +preserve a client-supplied string's original shape and character length for +this decision; normalizing that string into a one-item array MUST NOT add the +array envelope to its boundary calculation. An internal HTTP bridge +owner-forward hop MUST preserve that original string shape so the owner's +request validation reaches the same classification as the origin. +During a rolling upgrade, when an older origin forwards only a normalized +one-item array and the owner cannot validate the additive exact-body signature, +the owner MUST use conservative canonical-shape precedence. An exact canonical +normalized raw-string shape (`role=user` with one `input_text` part) MUST be +classified by its contained text length, not by its array or item serialization. +That wire shape is byte-identical to a genuine client array of the same form, so +the owner cannot recover the original provenance; the contained-text rule +therefore applies to both origins. A noncanonical one-item array MUST retain the +legacy compact-item predicate. Neither path may count a normalization envelope +as client text. In the +inverse rolling-upgrade direction, an upgraded origin without positive proof +that the selected owner implements this classifier MUST NOT dispatch an input +whose current and legacy classifications disagree in either direction. This +guard MUST cover a client string below 4096 characters whose normalized item +reaches 4096 compact-serialization characters, a multi-item array containing +only the allowed tool-output item types, a single tool output whose legacy +item serialization reaches 4096 characters, and a one-item array whose whole-array +serialization reaches 4096 characters while its item serialization does not. +It MUST also cover full-resend-shaped system/developer-only arrays that +normalize to empty input. Truly empty input and small single-message arrays +whose current and legacy classifications are both delta-only MUST remain +outside this upgrade requirement. +The origin MUST fail +closed or enter an already-authorized local recovery path before owner I/O. +For a request with turn state and no client `previous_response_id`, the typed +`owner_forward` / `owner_input_shape_upgrade_required` failure MUST be eligible +for the existing local turn-state takeover path. Takeover MUST still require a +successful fresh durable lookup with no active owner lease and the existing +continuity-routing checks. A failed lookup or active lease MUST fail closed; +missing owner capability proof MUST NOT permit owner dispatch. +A proxy-injected durable anchor MUST NOT count as a client-supplied previous +response id for this decision. The origin MUST resolve fresh turn-state +ownership without using that injected anchor as a lookup alias, and any retained +anchor MUST remain constrained to its original account. +Positive proof MUST come from a live bridge-ring advertisement containing the +exact input-shape-classifier capability and a process epoch equal to the +durable owner's recorded `owner_process_epoch`. When that proof matches, the +origin MAY dispatch the classification-ambiguous shape to the upgraded owner. +Missing, malformed, stale, or epoch-mismatched advertisements MUST NOT +authorize dispatch, including an advertisement left by an earlier process +that reused the same instance id. + +Capability-gated forwards MUST carry `x-codex-bridge-owner-process-epoch` +with the proven process epoch, authenticated by the exact-body signature. +The receiving owner MUST reject a signed epoch unequal to its local process +epoch before continuity selection. A nonempty process epoch MUST NOT be +authorized by either the legacy primary proof or the public pre-input-shape V2 +proof, because neither codec authenticates that field. These forwards MUST carry only +`x-codex-bridge-input-shape-signature-v2` as their body proof and MUST NOT +include either the legacy primary signature or the public +`x-codex-bridge-signature-v2` proof. A predecessor process that ignores the +epoch therefore cannot accept the request after a rollback. + +An upgraded owner-forward request MUST advertise +`x-codex-bridge-input-shape-version: 2` when it posts a body whose exact input +shape is known. The exact posted body, marker value, and optional owner process +epoch MUST be authenticated by the independent +`x-codex-bridge-input-shape-signature-v2` proof. +The owner MUST trust the current-shape mode only when that signature validates +with the same header value; a missing, malformed, or primary-signature-only +marker MUST keep legacy compatibility classification. Receivers implementing +this versioned input-shape contract MUST reject an unsupported nonempty +version before the forwarded request reaches continuity selection. + +For ordinary forwards, `x-codex-bridge-signature-v2` MUST retain the public +pre-input-shape codec: it signs `model_dump_for_forwarding()` and MUST NOT add +the input-shape marker or owner process epoch to its structured payload. The +origin MUST send this proof alongside the independent exact-shape proof so a +pre-input-shape owner can authenticate file-bound forwards without using the +legacy primary fallback. An upgraded receiver MUST also accept the previously +deployed shape-v2 layout in which the exact-shape proof bytes were carried in +`x-codex-bridge-signature-v2`; this acceptance MUST still require the signed +marker and, when present, the matching local owner process epoch. + +The marker advertises the origin's input-shape provenance, not the selected +owner's capability. For inputs whose current and legacy classifications agree, +forwarding MUST NOT require classifier capability proof and MUST retain the +existing primary-signature fallback, subject to its existing restrictions. +A predecessor owner may ignore the additive marker and exact-shape proof and +validate the public full-context proof, including for file-bound requests. +The origin MUST NOT remove a known-shape marker merely because the destination +lacks classifier capability proof. + +A payload already classified under legacy forwarding MUST omit the marker +when forwarded again. Its exact-body signature MUST bind the posted body +without an input-shape version. A current receiver accepting that signature +MUST retain legacy compatibility classification. This MUST NOT bypass the +capability and process-epoch checks for classification-ambiguous requests. + +#### Scenario: Large single tool output retains its continuation anchor + +- **GIVEN** a quarantined session has a durable turn-state anchor +- **AND** a request contains one tool output whose serialized size exceeds 4096 characters +- **AND** the client omits `previous_response_id` +- **WHEN** the bridge classifies the request +- **THEN** it MUST treat the input as delta-only and retain the durable anchor +- **AND** forwarding to an owner with a disagreeing legacy classifier MUST require the existing capability and process-epoch proof + +#### Scenario: Quarantine preserves durable context for delta-only requests + +- **GIVEN** a live bridge session is quarantined and its durable anchor is + available +- **WHEN** a genuine delta-only continuation arrives for that session key +- **THEN** the quarantined live session is excluded from local reuse and + full-resend anchor injection +- **AND** the request still resolves and receives its durable anchor +- **AND** no account health, routing, or durable ownership state changes + +#### Scenario: Legacy owner forwarding uses canonical normalized text length + +- **GIVEN** an older origin normalized a below-boundary client string into a + one-item array before forwarding it to a newer owner +- **AND** the forward validates only through the rolling-upgrade legacy + signature fallback +- **WHEN** the newer owner classifies the request shape +- **THEN** it MUST classify the exact canonical normalized shape by its + contained text length, not by its normalization envelope +- **AND** it MUST retain the durable previous-response anchor + +#### Scenario: Legacy fallback uses the compact-item predicate for noncanonical arrays + +- **GIVEN** an older origin forwards a genuinely noncanonical one-item array + such as `["x" * 4094]` +- **AND** the forward validates only through the rolling-upgrade legacy + signature fallback +- **WHEN** the newer owner classifies the request shape +- **THEN** it MUST use the compact serialization of that item for the 4096-character + boundary +- **AND** it MUST classify the item as full-resend-shaped at exactly 4096 + characters + +#### Scenario: Unauthenticated input-shape marker stays legacy + +- **GIVEN** a forwarded body carries `x-codex-bridge-input-shape-version: 2` +- **AND** its exact-body signature is missing or does not bind that marker +- **WHEN** the owner validates the primary bridge signature +- **THEN** it MUST accept only under legacy compatibility classification +- **AND** it MUST NOT infer current-shape mode from the marker alone + +#### Scenario: Owner is replaced after capability proof + +- **GIVEN** an origin proved the selected owner's process epoch and classifier + capability for an ambiguous delta-only request +- **WHEN** a replacement process receives the forward at the same instance id +- **THEN** an upgraded replacement MUST reject the signed process-epoch mismatch +- **AND** a predecessor replacement MUST fail signature validation without a + legacy primary fallback +- **AND** neither replacement may select continuity or suppress the durable anchor + +#### Scenario: Current origin does not expose a delta to a legacy owner + +- **GIVEN** an upgraded origin selects a remote owner whose current classifier + capability is not positively known +- **AND** the request is delta-only under the current classifier but full-resend + shaped after legacy normalization +- **WHEN** the origin reaches the owner-forward boundary +- **THEN** it MUST NOT dispatch the request to that owner +- **AND** it MUST fail closed or use an already-authorized local recovery path + before the legacy owner can suppress the durable anchor + +#### Scenario: Proven upgraded owner receives an ambiguous delta + +- **GIVEN** an upgraded origin selects a live remote owner +- **AND** the ring advertises the exact input-shape-classifier capability with + a process epoch equal to the durable owner's recorded process epoch +- **AND** the request is delta-only under the current classifier but + full-resend shaped after legacy normalization +- **WHEN** the origin reaches the owner-forward boundary +- **THEN** it MAY dispatch the request to that owner +- **AND** the owner MUST retain the durable previous-response anchor + +#### Scenario: Replaced owner process cannot inherit capability proof + +- **GIVEN** an instance id has a classifier-capable ring advertisement from an + earlier owner process +- **AND** the durable owner record names a different current process epoch +- **WHEN** an upgraded origin evaluates an ambiguous delta-only owner forward +- **THEN** the stale advertisement MUST NOT authorize dispatch +- **AND** the origin MUST fail closed or use an already-authorized local + recovery path before owner I/O + ### Requirement: HTTP bridge streams emit downstream liveness frames while pending When an HTTP bridge Responses request is waiting for upstream queue events, the system MUST emit a downstream SSE liveness frame at the configured `sse_keepalive_interval_seconds` interval so downstream clients do not disconnect before the upstream terminal frame arrives. The interval is dashboard-managed: a non-NULL `dashboard_settings.sse_keepalive_interval_seconds` MUST override the environment value, `0` disables generated liveness frames, and the value MUST be read from the `SettingsCache` snapshot bound to the request rather than from the environment alone. The first generated liveness frame MUST be delayed until after the HTTP bridge startup-error probe window so a local startup `ProxyResponseError` can still be surfaced as a non-2xx HTTP response. Once a generated liveness frame is emitted, the stream MUST be considered started for later HTTP-error propagation decisions, so a subsequent upstream `response.failed` is forwarded in-stream instead of being raised as a startup HTTP error. If the pending request already has a response id, the liveness frame MAY be a `response.in_progress` SSE event for that response id. If no response id is known yet, the Codex CLI route MUST emit an ignored `codex.keepalive` SSE data event because comment-only frames do not reset the CLI's EventSource idle timer. Public `/v1/responses` stream normalization MUST preserve SSE comment keepalives instead of treating them as malformed data, and MUST drop `codex.*` liveness events from the public OpenAI SDK contract surface. @@ -10830,4 +11051,3 @@ still awaiting I/O. - **WHEN** the cooldown expires and the next full-resend request is admitted as the probe - **THEN** the key is quarantined and the probe is planned without the dead anchor - **AND** the probe resends full history rather than the dead anchor - diff --git a/tests/integration/test_daybreak_capability_routes.py b/tests/integration/test_daybreak_capability_routes.py index b8d66512b3..0bfc9bd146 100644 --- a/tests/integration/test_daybreak_capability_routes.py +++ b/tests/integration/test_daybreak_capability_routes.py @@ -456,7 +456,7 @@ async def fail_before_bridge_routing(*_args: Any, **_kwargs: Any) -> None: response = await async_client.post( "/internal/bridge/responses", headers=headers, - json=payload.model_dump_for_forwarding(), + json=payload.model_dump_for_http_bridge_owner_forwarding(), ) if auth_state == "valid": @@ -917,7 +917,7 @@ async def fail_before_bridge_routing(*_args: Any, **_kwargs: Any) -> None: response = await async_client.post( "/internal/bridge/responses", headers=headers, - json=payload.model_dump_for_forwarding(), + json=payload.model_dump_for_http_bridge_owner_forwarding(), ) assert response.status_code == 400 diff --git a/tests/integration/test_http_bridge_input_shape.py b/tests/integration/test_http_bridge_input_shape.py new file mode 100644 index 0000000000..ac055e9967 --- /dev/null +++ b/tests/integration/test_http_bridge_input_shape.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import socket + +import pytest + +from app.dependencies import get_proxy_service_for_app +from app.modules.proxy._service.http_bridge import quarantine +from app.modules.proxy.load_balancer import AccountSelection +from tests.integration import test_http_responses_bridge as bridge + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +@pytest.mark.parametrize("input_shape", ["parallel_outputs", "short_raw_string", "large_single_output"]) +async def test_quarantined_http_continuation_keeps_durable_anchor(async_client, app_instance, monkeypatch, input_shape): + bridge._install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) + account_id = await bridge._import_account(async_client, "shape-account", "shape@example.com") + account = await bridge._get_account(account_id) + upstreams = [ + bridge._FakeBridgeUpstreamWebSocket("resp_shape_first"), + bridge._FakeBridgeUpstreamWebSocket("resp_shape_next"), + ] + connected = [] + + async def select_account(self, deadline, **kwargs): + return AccountSelection(account=account, error_message=None, error_code=None) + + async def ensure_fresh(self, target, *, force=False, timeout_seconds): + return target + + async def connect(headers, access_token, account_id_header, *, base_url=None, session=None): + upstream = upstreams[len(connected)] + connected.append(upstream) + return upstream + + monkeypatch.setattr(bridge.proxy_module.ProxyService, "_select_account_with_budget", select_account) + monkeypatch.setattr(bridge.proxy_module.ProxyService, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(bridge.proxy_module, "connect_responses_websocket", connect) + headers = {"x-codex-session-id": "shape-session", "x-codex-turn-state": "shape-turn"} + service = get_proxy_service_for_app(app_instance) + try: + first = await async_client.post( + "/v1/responses", json={"model": "gpt-5.1", "input": "first question"}, headers=headers + ) + assert first.status_code == 200, first.text + first_id = first.json()["id"] + session = next(iter(service._http_bridge_sessions.values())) + quarantine._quarantine_http_bridge_session( + service, session, reason=quarantine._HTTP_BRIDGE_QUARANTINE_WEDGED_REATTACH_REASON + ) + continuation = ( + [ + {"type": "function_call_output", "call_id": "call_a", "output": "a"}, + {"type": "function_call_output", "call_id": "call_b", "output": "b"}, + ] + if input_shape == "parallel_outputs" + else "x" * 4095 + ) + if input_shape == "large_single_output": + continuation = [{"type": "function_call_output", "call_id": "call_a", "output": "x" * 4096}] + response = await async_client.post( + "/v1/responses", json={"model": "gpt-5.1", "input": continuation}, headers=headers + ) + assert response.status_code == 200, response.text + assert len(connected) == 2 + sent = json.loads(upstreams[1].sent_text[0]) + assert sent["previous_response_id"] == first_id + if input_shape in {"parallel_outputs", "large_single_output"}: + assert sent["input"] == continuation + else: + assert sent["input"] == [{"role": "user", "content": [{"type": "input_text", "text": continuation}]}] + finally: + for live in list(service._http_bridge_sessions.values()): + await service._close_http_bridge_session(live) diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 2bd191a08a..56645d892d 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -5190,7 +5190,7 @@ async def test_forwarded_priority_prompt_cache_mismatch_forks_on_canonical_owner ): from app.core.middleware import request_id as request_id_middleware_module from app.modules.proxy import api as proxy_api_module - from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_request owner_settings = _make_app_settings( enabled=True, @@ -5338,7 +5338,8 @@ def capture_scheduled_sessions(sessions, *, reason): original_affinity_kind=canonical_key.affinity_kind, original_affinity_key=canonical_key.affinity_key, ) - forward_headers = build_owner_forward_headers( + owner_request = build_owner_forward_request( + body=priority_payload.model_dump_for_http_bridge_owner_forwarding(), headers={"x-request-id": "forwarded-priority-request"}, payload=priority_payload, context=forward_context, @@ -5356,8 +5357,8 @@ def capture_scheduled_sessions(sessions, *, reason): priority_response = await async_client.post( "/internal/bridge/responses", - json=priority_payload.model_dump_for_forwarding(), - headers=forward_headers, + json=owner_request.body, + headers=owner_request.headers, ) assert priority_response.status_code == 200, priority_response.text @@ -5385,7 +5386,7 @@ async def test_forwarded_recovery_uses_durable_owner_and_strips_stale_affinity( ): from app.modules.proxy import api as proxy_api_module from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key - from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_headers + from app.modules.proxy.http_bridge_forwarding import HTTPBridgeForwardContext, build_owner_forward_request target_settings = _make_app_settings(enabled=True, instance_id="instance-b") _install_proxy_settings( @@ -5499,7 +5500,8 @@ async def fail_legacy_stream(*args, **kwargs): original_affinity_kind=recovery_kind, original_affinity_key=recovery_key, ) - forward_headers = build_owner_forward_headers( + owner_request = build_owner_forward_request( + body=payload.model_dump_for_http_bridge_owner_forwarding(), headers={ "session_id": "stale-session", "session-id": "stale-session-dash", @@ -5516,8 +5518,8 @@ async def fail_legacy_stream(*args, **kwargs): response = await asyncio.wait_for( async_client.post( "/internal/bridge/responses", - json=payload.model_dump_for_forwarding(), - headers=forward_headers, + json=owner_request.body, + headers=owner_request.headers, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) @@ -13630,10 +13632,12 @@ async def fake_connect_responses_websocket( @pytest.mark.asyncio +@pytest.mark.parametrize("reconnect_delay_seconds", [0.0, 0.25]) async def test_v1_responses_http_bridge_idle_recovery_hands_reader_to_replacement( async_client, app_instance, monkeypatch, + reconnect_delay_seconds: float, ): app_settings = _make_app_settings(enabled=True) app_settings.sse_keepalive_interval_seconds = 0.01 @@ -13643,7 +13647,6 @@ async def test_v1_responses_http_bridge_idle_recovery_hands_reader_to_replacemen dashboard_settings=_make_dashboard_settings(), ) monkeypatch.setattr(proxy_module, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.01) - monkeypatch.setattr(proxy_module, "_STREAM_KEEPALIVE_MAX_COUNT", 1) account_id = await _import_account( async_client, "acc_http_bridge_reader_handoff", @@ -13707,6 +13710,8 @@ async def fake_connect_responses_websocket( ): del headers, access_token, account_id_header, base_url, session nonlocal connect_count + if connect_count == 1: + await asyncio.sleep(reconnect_delay_seconds) upstream = upstreams[connect_count] connect_count += 1 return upstream @@ -13715,6 +13720,24 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) service = get_proxy_service_for_app(app_instance) + next_receive_timeout = service._next_websocket_receive_timeout + + async def short_reader_timeout( + pending_requests: deque[proxy_module._WebSocketRequestState], + *, + pending_lock: anyio.Lock, + proxy_request_budget_seconds: float, + stream_idle_timeout_seconds: float, + ) -> proxy_module._WebSocketReceiveTimeout | None: + # Trigger the silent reader without shrinking the downstream retry budget. + return await next_receive_timeout( + pending_requests, + pending_lock=pending_lock, + proxy_request_budget_seconds=proxy_request_budget_seconds, + stream_idle_timeout_seconds=0.1, + ) + + monkeypatch.setattr(service, "_next_websocket_receive_timeout", short_reader_timeout) record_retry_circuit_failure = AsyncMock(wraps=service._record_http_bridge_retry_circuit_failure) monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_retry_circuit_failure) diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index a3377222fb..ceee3ae575 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -159,7 +159,7 @@ async def fail_finalize(self, reservation_id: str, **kwargs: object) -> None: response = await async_client.post( "/internal/bridge/responses", - json=forwarded_payload.model_dump_for_forwarding(), + json=forwarded_payload.model_dump_for_http_bridge_owner_forwarding(), headers=headers, ) @@ -1817,7 +1817,7 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles response = await async_client.post( "/internal/bridge/responses", - json=forwarded_payload.model_dump_for_forwarding(), + json=forwarded_payload.model_dump_for_http_bridge_owner_forwarding(), headers=headers, ) diff --git a/tests/unit/test_http_bridge_forwarding.py b/tests/unit/test_http_bridge_forwarding.py index 37034a6fb7..3816f3fee1 100644 --- a/tests/unit/test_http_bridge_forwarding.py +++ b/tests/unit/test_http_bridge_forwarding.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from collections.abc import AsyncIterator, Iterator from pathlib import Path from types import SimpleNamespace @@ -10,9 +11,12 @@ import pytest from aiohttp.client_reqrep import ConnectionKey +from app.core.clients.proxy import ProxyResponseError from app.core.config.settings import get_settings +from app.core.crypto import get_or_create_key from app.core.openai.requests import ResponsesRequest from app.modules.api_keys.service import ApiKeyUsageReservationData +from app.modules.proxy._service.http_bridge import helpers as http_bridge_helpers_module from app.modules.proxy.http_bridge_forwarding import ( HTTP_BRIDGE_AFFINITY_KEY_HEADER, HTTP_BRIDGE_AFFINITY_KIND_HEADER, @@ -21,8 +25,11 @@ HTTP_BRIDGE_CODEX_AFFINITY_HEADER, HTTP_BRIDGE_FILE_OWNER_HEADER, HTTP_BRIDGE_FORWARDED_HEADER, + HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER, + HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER, HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER, HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER, + HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER, HTTP_BRIDGE_RESERVATION_ID_HEADER, HTTP_BRIDGE_RESERVATION_KEY_ID_HEADER, HTTP_BRIDGE_RESERVATION_MODEL_HEADER, @@ -32,6 +39,7 @@ HTTP_BRIDGE_TARGET_INSTANCE_HEADER, HTTPBridgeForwardContext, HTTPBridgeOwnerClient, + _bridge_forward_input_shape_signature, _bridge_forward_signature, _bridge_forward_tools_bound_signature, _iter_sse_event_blocks, @@ -39,6 +47,7 @@ _owner_forward_timeout, _OwnerForwardStreamTimeoutError, build_owner_forward_headers, + build_owner_forward_request, parse_forwarded_request, ) from tests.simulation.virtual_time import VirtualClock, VirtualScheduler @@ -82,6 +91,8 @@ def _use_legacy_forward_signature( ) -> None: headers.pop(HTTP_BRIDGE_SIGNATURE_VERSION_HEADER, None) headers.pop(HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER, None) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER, None) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER, None) # A genuinely pre-#1203 origin sends no tamper-proofing header, so the # receiver must exercise the primary-signature fallback rather than the # tamper-proofing fast path. @@ -99,6 +110,228 @@ def _use_legacy_forward_signature( ) +def _frozen_upstream_v1_tools_bound_signature( + *, + payload: ResponsesRequest, + context: HTTPBridgeForwardContext, + signature_version: str | None = None, +) -> str: + """Independent public codec from before the input-shape change.""" + + import hashlib + import hmac + + body_json = json.dumps( + payload.model_dump_for_forwarding(), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + body_digest = hashlib.sha256(body_json.encode("utf-8")).hexdigest() + signing_payload = json.dumps( + { + "body_digest": body_digest, + "client_ip": context.client_ip, + "client_ip_present": context.client_ip is not None, + "codex_session_affinity": context.codex_session_affinity, + "downstream_turn_state": context.downstream_turn_state, + "file_owner_account_id": context.file_owner_account_id, + "include_client_ip": True, + "origin_instance": context.origin_instance, + "original_affinity_key": context.original_affinity_key, + "original_affinity_kind": context.original_affinity_kind, + "original_request_unanchored": context.original_request_unanchored, + "protocol": "codex-lb-http-bridge-forward-tools-bound", + "reservation": ( + { + "id": context.reservation.reservation_id, + "key_id": context.reservation.key_id, + "model": context.reservation.model, + } + if context.reservation is not None + else None + ), + "signature_version": signature_version, + "target_instance": context.target_instance, + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + secret = get_or_create_key(get_settings().encryption_key_file) + return hmac.new(secret, signing_payload.encode("utf-8"), hashlib.sha256).hexdigest() + + +def _frozen_deployed_shape_v2_signature( + *, + payload: ResponsesRequest, + context: HTTPBridgeForwardContext, + input_shape_version: str | None = "2", +) -> str: + """Independent codec from deployed 9ede3db, before the header split.""" + + import hashlib + import hmac + + body_json = json.dumps( + payload.model_dump_for_http_bridge_owner_forwarding(), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + body_digest = hashlib.sha256(body_json.encode("utf-8")).hexdigest() + fields: dict[str, object] = { + "body_digest": body_digest, + "client_ip": context.client_ip, + "client_ip_present": context.client_ip is not None, + "codex_session_affinity": context.codex_session_affinity, + "downstream_turn_state": context.downstream_turn_state, + "file_owner_account_id": context.file_owner_account_id, + "include_client_ip": True, + "origin_instance": context.origin_instance, + "original_affinity_key": context.original_affinity_key, + "original_affinity_kind": context.original_affinity_kind, + "original_request_unanchored": context.original_request_unanchored, + "protocol": "codex-lb-http-bridge-forward-tools-bound", + "reservation": None, + "signature_version": None, + "target_instance": context.target_instance, + } + if input_shape_version is not None: + fields["input_shape_version"] = input_shape_version + if context.expected_owner_process_epoch is not None: + fields["expected_owner_process_epoch"] = context.expected_owner_process_epoch + signing_payload = json.dumps(fields, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + secret = get_or_create_key(get_settings().encryption_key_file) + return hmac.new(secret, signing_payload.encode("utf-8"), hashlib.sha256).hexdigest() + + +@pytest.mark.parametrize("with_file", [False, True]) +def test_upstream_v1_codec_replays_both_directions(with_file: bool) -> None: + payload = _payload_with_file() if with_file else _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=True, + downstream_turn_state="http_turn_cross_version", + file_owner_account_id="acc-file-owner" if with_file else None, + ) + old_signature = _frozen_upstream_v1_tools_bound_signature(payload=payload, context=context) + + new_headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + assert new_headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] == old_signature + + old_headers = { + HTTP_BRIDGE_FORWARDED_HEADER: "1", + HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER: context.origin_instance, + HTTP_BRIDGE_TARGET_INSTANCE_HEADER: context.target_instance, + HTTP_BRIDGE_CODEX_AFFINITY_HEADER: "1", + HTTP_BRIDGE_SIGNATURE_V2_HEADER: old_signature, + "x-codex-turn-state": "http_turn_cross_version", + } + if context.file_owner_account_id is not None: + old_headers[HTTP_BRIDGE_FILE_OWNER_HEADER] = context.file_owner_account_id + + forwarded, error = parse_forwarded_request(old_headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert forwarded.context == context + assert payload._codex_lb_legacy_owner_forwarding_input_shape is True + + +def test_public_v2_proof_does_not_authorize_injected_owner_epoch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + + # An ordinary predecessor proof remains a valid positive control when no + # process epoch is claimed. + forwarded, error = parse_forwarded_request( + headers, + payload=payload, + current_instance="instance-b", + ) + assert error is None + assert forwarded is not None + + # The public V2 codec intentionally does not authenticate owner epoch. An + # injected epoch must therefore require the independent exact-shape proof, + # even when its value happens to name the current process. + monkeypatch.setattr( + "app.modules.proxy.http_bridge_forwarding.http_bridge_owner_process_epoch", + lambda: "unproven-process", + ) + headers[HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER] = "unproven-process" + forged_payload = _payload() + forwarded, error = parse_forwarded_request( + headers, + payload=forged_payload, + current_instance="instance-b", + ) + assert forwarded is None + assert error is not None + assert error.payload["error"]["code"] == "bridge_forward_invalid" + + +@pytest.mark.parametrize("with_epoch", [False, True]) +def test_deployed_shape_v2_origin_is_accepted_after_header_split( + monkeypatch: pytest.MonkeyPatch, + with_epoch: bool, +) -> None: + payload = _payload_with_file() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=True, + downstream_turn_state="http_turn_shape_v2", + file_owner_account_id="acc-file-owner", + expected_owner_process_epoch="owner-process" if with_epoch else None, + ) + monkeypatch.setattr( + "app.modules.proxy.http_bridge_forwarding.http_bridge_owner_process_epoch", + lambda: "owner-process", + ) + deployed_signature = _frozen_deployed_shape_v2_signature(payload=payload, context=context) + candidate_headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + assert candidate_headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == deployed_signature + if with_epoch: + assert HTTP_BRIDGE_SIGNATURE_V2_HEADER not in candidate_headers + else: + # A deployed 9ede3db owner reads only this old header and cannot verify + # the split sender's file-bound proof. The cutover must update all + # owners together; the inverse old-origin -> new-owner path is accepted + # below. + assert candidate_headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] != deployed_signature + headers = { + HTTP_BRIDGE_FORWARDED_HEADER: "1", + HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER: context.origin_instance, + HTTP_BRIDGE_TARGET_INSTANCE_HEADER: context.target_instance, + HTTP_BRIDGE_CODEX_AFFINITY_HEADER: "1", + HTTP_BRIDGE_FILE_OWNER_HEADER: "acc-file-owner", + HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER: "2", + HTTP_BRIDGE_SIGNATURE_V2_HEADER: deployed_signature, + "x-codex-turn-state": "http_turn_shape_v2", + } + if with_epoch: + headers[HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER] = "owner-process" + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert forwarded.context == context + assert payload._codex_lb_input_shape_wire_version == "2" + + def test_parse_forwarded_request_accepts_signed_internal_forward() -> None: payload = _payload() context = HTTPBridgeForwardContext( @@ -163,6 +396,7 @@ def test_parse_forwarded_request_rejects_unbound_file_owner_proof(downgrade: str if downgrade == "tamper": headers[HTTP_BRIDGE_FILE_OWNER_HEADER] = "acc-attacker" else: + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) headers.pop(HTTP_BRIDGE_SIGNATURE_V2_HEADER) forwarded, error = parse_forwarded_request( @@ -186,6 +420,7 @@ def test_parse_forwarded_request_rejects_file_payload_without_full_context_signa ) headers = build_owner_forward_headers(headers={}, payload=payload, context=context) headers.pop(HTTP_BRIDGE_FILE_OWNER_HEADER, None) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) headers.pop(HTTP_BRIDGE_SIGNATURE_V2_HEADER) forwarded, error = parse_forwarded_request( @@ -242,11 +477,11 @@ def test_parse_forwarded_request_rejects_body_with_injected_empty_tools() -> Non tampered_body["tools"] = [] tampered_payload = ResponsesRequest.model_validate(tampered_body) assert "tools" in tampered_payload.model_fields_set - assert headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] == _bridge_forward_tools_bound_signature( + assert headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == _bridge_forward_input_shape_signature( payload=payload, context=context, ) - assert headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] != _bridge_forward_tools_bound_signature( + assert headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] != _bridge_forward_input_shape_signature( payload=tampered_payload, context=context, ) @@ -323,6 +558,8 @@ def test_parse_forwarded_request_accepts_legacy_forward_with_spoofed_v2_header() downstream_turn_state=None, ) headers = build_owner_forward_headers(headers={}, payload=old_origin_payload, context=context) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER) headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] = "spoofed-by-external-client" forwarded, error = parse_forwarded_request( @@ -347,6 +584,7 @@ def test_build_owner_forward_headers_drops_client_supplied_bridge_headers() -> N downstream_turn_state=None, ) inbound = { + "x-codex-bridge-input-shape-signature-v2": "client-spoofed", "x-codex-bridge-signature-v2": "client-spoofed", "x-codex-bridge-signature": "client-spoofed", "x-codex-bridge-future-unknown": "client-spoofed", @@ -358,6 +596,10 @@ def test_build_owner_forward_headers_drops_client_supplied_bridge_headers() -> N payload=payload, context=context, ) + assert headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == _bridge_forward_input_shape_signature( + payload=payload, + context=context, + ) assert headers[HTTP_BRIDGE_SIGNATURE_HEADER] != "client-spoofed" assert "x-codex-bridge-future-unknown" not in headers assert headers["x-openai-client-version"] == "1.2.3" @@ -408,6 +650,7 @@ def test_parse_forwarded_request_falls_back_to_legacy_signature_without_v2() -> ) headers = build_owner_forward_headers(headers={}, payload=old_origin_payload, context=context) del headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] + del headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] forwarded, error = parse_forwarded_request( headers, @@ -419,6 +662,376 @@ def test_parse_forwarded_request_falls_back_to_legacy_signature_without_v2() -> assert forwarded.context == context +def test_parse_forwarded_request_authenticated_input_shape_v2_selects_current_mode() -> None: + payload = _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + + assert headers[HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER] == "2" + assert headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == _bridge_forward_input_shape_signature( + payload=payload, + context=context, + input_shape_version="2", + ) + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert payload._codex_lb_input_shape_wire_version == "2" + assert payload._codex_lb_legacy_owner_forwarding_input_shape is False + + +def test_parse_forwarded_request_predecessor_v2_normalized_array_stays_legacy() -> None: + raw_text = "x" * 4035 + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": raw_text}]}], + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + # A predecessor knows the v2 body-bound signature but predates the + # additive input-shape marker. Recompute that signature over the same + # normalized array with no marker to model its wire format exactly. + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] = _frozen_deployed_shape_v2_signature( + payload=payload, + context=context, + input_shape_version=None, + ) + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert payload._codex_lb_input_shape_wire_version is None + assert payload._codex_lb_legacy_owner_forwarding_input_shape is True + # The normalized array is exactly the canonical raw-string shape; its + # envelope must not count toward the full-resend boundary. + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is False + + +@pytest.mark.parametrize("tamper", ["missing", "mismatched"]) +def test_parse_forwarded_request_untrusted_input_shape_marker_stays_legacy(tamper: str) -> None: + payload = _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + if tamper == "missing": + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + else: + headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] = _bridge_forward_input_shape_signature( + payload=payload, + context=context, + input_shape_version=None, + ) + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert payload._codex_lb_input_shape_wire_version is None + assert payload._codex_lb_legacy_owner_forwarding_input_shape is True + + +@pytest.mark.parametrize("drop_signatures", [False, True]) +def test_parse_forwarded_request_rejects_unsupported_input_shape_version(drop_signatures: bool) -> None: + payload = _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + headers[HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER] = "3" + if drop_signatures: + headers.pop(HTTP_BRIDGE_SIGNATURE_HEADER) + headers.pop(HTTP_BRIDGE_SIGNATURE_V2_HEADER) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert forwarded is None + assert error is not None + assert error.status_code == 400 + assert error.payload["error"].get("code") == "bridge_forward_invalid" + + +def test_legacy_owner_forwarding_omits_input_shape_version_marker() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": "continue"}], + "tools": [], + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + _use_legacy_forward_signature(headers, payload=payload, context=context) + + forwarded, error = parse_forwarded_request(headers, payload=payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert payload._codex_lb_legacy_owner_forwarding_input_shape is True + + legacy_headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + assert HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER not in legacy_headers + assert legacy_headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] == _bridge_forward_tools_bound_signature( + payload=payload, + context=context, + ) + assert legacy_headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == _bridge_forward_input_shape_signature( + payload=payload, + context=context, + input_shape_version=None, + ) + + +def test_raw_string_and_canonical_array_collision_is_separated_by_shape_proof() -> None: + raw_text = "x" * 4035 + canonical_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": raw_text}], + } + ] + raw_payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": raw_text}) + canonical_payload = ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": canonical_input} + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + + assert raw_payload.input == canonical_payload.input + assert raw_payload.model_dump_for_forwarding() == canonical_payload.model_dump_for_forwarding() + assert _bridge_forward_signature(payload=raw_payload, context=context) == _bridge_forward_signature( + payload=canonical_payload, + context=context, + ) + + raw_owner_wire = raw_payload.model_dump_for_http_bridge_owner_forwarding() + canonical_owner_wire = canonical_payload.model_dump_for_http_bridge_owner_forwarding() + assert raw_owner_wire["input"] == raw_text + assert canonical_owner_wire["input"] == canonical_input + assert raw_owner_wire != canonical_owner_wire + assert _bridge_forward_input_shape_signature( + payload=raw_payload, + context=context, + input_shape_version="2", + ) != _bridge_forward_input_shape_signature( + payload=canonical_payload, + context=context, + input_shape_version="2", + ) + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(raw_payload) is False + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(canonical_payload) is True + + +def test_owner_forward_request_builds_one_authenticated_body_header_pair() -> None: + payload = _payload() + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + + owner_request = build_owner_forward_request( + body=payload.model_dump_for_http_bridge_owner_forwarding(), + headers={}, + payload=payload, + context=context, + ) + + assert owner_request.body == payload.model_dump_for_http_bridge_owner_forwarding() + assert owner_request.headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] == _bridge_forward_input_shape_signature( + payload=payload, + context=context, + ) + + +@pytest.mark.parametrize("tamper", ["body", "shape-signature"]) +def test_epoch_bound_shape_proof_rejects_transplant( + monkeypatch: pytest.MonkeyPatch, + tamper: str, +) -> None: + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "x" * 4095}) + other_payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "x" * 4095}]}], + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + expected_owner_process_epoch="owner-process", + ) + monkeypatch.setattr( + "app.modules.proxy.http_bridge_forwarding.http_bridge_owner_process_epoch", + lambda: "owner-process", + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + received_payload = other_payload if tamper == "body" else payload + if tamper == "shape-signature": + headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] = _bridge_forward_input_shape_signature( + payload=other_payload, + context=context, + ) + + forwarded, error = parse_forwarded_request( + headers, + payload=received_payload, + current_instance="instance-b", + ) + + assert forwarded is None + assert error is not None + assert error.status_code == 400 + + +@pytest.mark.parametrize( + ("input_length", "expected"), + [ + pytest.param(4035, False, id="canonical-array-envelope-reaches-boundary"), + pytest.param(4095, False, id="canonical-text-below-boundary"), + pytest.param(4096, True, id="canonical-text-at-boundary"), + ], +) +def test_legacy_owner_forward_preserves_canonical_raw_string_boundary(input_length: int, expected: bool) -> None: + """Legacy fallback uses contained text for the canonical normalized shape.""" + normalized_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "x" * input_length}], + } + ] + old_origin_payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": normalized_input, + "tools": [], + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=old_origin_payload, context=context) + _use_legacy_forward_signature(headers, payload=old_origin_payload, context=context) + + owner_payload = ResponsesRequest.model_validate(old_origin_payload.model_dump_for_forwarding()) + forwarded, error = parse_forwarded_request(headers, payload=owner_payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + compact_input = json.dumps(normalized_input, ensure_ascii=True, separators=(",", ":")) + assert len(compact_input) >= 4096 + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(owner_payload) is expected + + +@pytest.mark.parametrize( + ("input_length", "expected"), + [ + pytest.param(4035, False, id="raw-string-below-boundary"), + pytest.param(4095, False, id="raw-string-below-boundary-near"), + pytest.param(4096, True, id="raw-string-at-boundary"), + ], +) +def test_body_bound_owner_forward_preserves_raw_string_boundary(input_length: int, expected: bool) -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "x" * input_length, + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + owner_payload = ResponsesRequest.model_validate(payload.model_dump_for_http_bridge_owner_forwarding()) + + forwarded, error = parse_forwarded_request(headers, payload=owner_payload, current_instance="instance-b") + + assert error is None + assert forwarded is not None + assert owner_payload._codex_lb_input_shape_wire_version == "2" + assert owner_payload._codex_lb_legacy_owner_forwarding_input_shape is False + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(owner_payload) is expected + + +def test_body_bound_owner_forward_keeps_one_item_array_classification() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": [ + { + "role": "user", + "content": [{"type": "input_text", "text": "x" * 4035}], + } + ], + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + + forwarded, error = parse_forwarded_request( + headers, + payload=payload, + current_instance="instance-b", + ) + + assert error is None + assert forwarded is not None + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is True + + def test_build_owner_forward_headers_uses_v2_signature_with_client_ip_header() -> None: payload = _payload() context = HTTPBridgeForwardContext( @@ -911,6 +1524,7 @@ def test_parse_forwarded_request_rejects_tampered_signature() -> None: # Without the v2 header (pre-v2 origin), the tampered legacy signature # is rejected by the fallback verification. headers.pop(HTTP_BRIDGE_SIGNATURE_V2_HEADER, None) + headers.pop(HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER, None) forwarded, error = parse_forwarded_request( headers, payload=payload, @@ -927,6 +1541,7 @@ def test_parse_forwarded_request_rejects_tampered_signature() -> None: # forward is rejected. headers = build_owner_forward_headers(headers={}, payload=payload, context=context) headers[HTTP_BRIDGE_SIGNATURE_V2_HEADER] = "bad-signature" + headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] = "bad-signature" headers[HTTP_BRIDGE_SIGNATURE_HEADER] = "bad-signature" forwarded, error = parse_forwarded_request( headers, @@ -1140,6 +1755,135 @@ def post(self, url: str, **kwargs: object) -> FakeResponse: assert skip_auto_headers == {"Accept", "Accept-Encoding"} +@pytest.mark.asyncio +@pytest.mark.parametrize( + "input_value", + [ + pytest.param("x" * 4095, id="raw-string-boundary"), + pytest.param( + [{"type": "function_call_output", "call_id": "call-1", "output": "x" * 4096}], + id="large-single-tool-output-delta", + ), + pytest.param(["x" * 4092], id="array-full-legacy-delta"), + pytest.param(["x" * 4093], id="array-full-legacy-delta-upper"), + pytest.param( + [ + {"type": "function_call_output", "call_id": "call-1", "output": "first"}, + {"type": "function_call_output", "call_id": "call-2", "output": "second"}, + ], + id="parallel-tool-output-delta", + ), + ], +) +async def test_owner_forward_blocks_shapes_that_legacy_owners_reclassify( + monkeypatch: pytest.MonkeyPatch, + input_value: object, +) -> None: + dispatched = False + + class UnexpectedSession: + def __init__(self, **_kwargs: object) -> None: + nonlocal dispatched + dispatched = True + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", UnexpectedSession) + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": input_value}) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + ) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _event in HTTPBridgeOwnerClient().stream_responses( + owner_endpoint="http://instance-b:2455", + payload=payload, + headers={}, + context=context, + request_started_at=10.0, + ): + pass + + assert dispatched is False + assert exc_info.value.status_code == 503 + error = exc_info.value.payload.get("error") + assert isinstance(error, dict) + assert error.get("code") == "bridge_owner_forward_failed" + assert exc_info.value.failure_detail == "owner_input_shape_upgrade_required" + + +@pytest.mark.asyncio +async def test_owner_forward_dispatches_ambiguous_delta_after_owner_capability_proof( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + class FakeResponse: + status = 200 + + async def __aenter__(self) -> "FakeResponse": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + @property + def content(self) -> SimpleNamespace: + async def _iter_chunked(_: int) -> AsyncIterator[bytes]: + if False: + yield b"" + return + + return SimpleNamespace(iter_chunked=_iter_chunked) + + class FakeSession: + def __init__(self, **_kwargs: object) -> None: + return None + + async def __aenter__(self) -> "FakeSession": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + return None + + def post(self, url: str, **kwargs: object) -> FakeResponse: + captured["url"] = url + captured["json"] = kwargs["json"] + return FakeResponse() + + monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) + clock = VirtualClock(monotonic_value=10.0) + payload = ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "x" * 4095}, + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=False, + downstream_turn_state=None, + expected_owner_process_epoch="owner-process-b", + ) + + events = [ + event + async for event in HTTPBridgeOwnerClient().stream_responses( + owner_endpoint="http://instance-b:2455", + payload=payload, + headers={}, + context=context, + request_started_at=10.0, + owner_supports_input_shape_classifier=True, + clock=clock, + ) + ] + + assert captured["url"] == "http://instance-b:2455/internal/bridge/responses" + assert captured["json"] == payload.model_dump_for_http_bridge_owner_forwarding() + assert len(events) == 1 + assert '"code":"stream_incomplete"' in events[0] + + @pytest.mark.asyncio async def test_owner_forward_allows_json_content_type_for_internal_post( monkeypatch: pytest.MonkeyPatch, @@ -1186,7 +1930,7 @@ def post(self, url: str, **kwargs: object) -> FakeResponse: monkeypatch.setattr("app.modules.proxy.http_bridge_forwarding.aiohttp.ClientSession", FakeSession) client = HTTPBridgeOwnerClient() - payload = _payload() + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) context = HTTPBridgeForwardContext( origin_instance="instance-a", target_instance="instance-b", @@ -1214,6 +1958,7 @@ def post(self, url: str, **kwargs: object) -> FakeResponse: headers = cast(dict[str, str], captured["headers"]) forwarded_json = cast(dict[str, object], captured["json"]) assert "tools" not in forwarded_json + assert forwarded_json["input"] == "hi" forwarded_payload = ResponsesRequest.model_validate(forwarded_json) assert "tools" not in forwarded_payload.model_fields_set assert "tools" not in forwarded_payload.to_payload() @@ -1225,6 +1970,11 @@ def post(self, url: str, **kwargs: object) -> FakeResponse: assert error is None assert forwarded is not None assert forwarded.context == context + normalized_body = ResponsesRequest.model_validate(payload.model_dump_for_forwarding()) + assert headers[HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER] != _bridge_forward_input_shape_signature( + payload=normalized_body, + context=context, + ) assert isinstance(headers, dict) assert "Content-Type" not in headers assert "content-type" not in headers diff --git a/tests/unit/test_http_bridge_forwarding_epoch.py b/tests/unit/test_http_bridge_forwarding_epoch.py new file mode 100644 index 0000000000..6c94ccd59a --- /dev/null +++ b/tests/unit/test_http_bridge_forwarding_epoch.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +import time +from collections.abc import Iterator +from dataclasses import replace +from pathlib import Path + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestServer + +from app.core.clients.proxy import ProxyResponseError +from app.core.config.settings import get_settings +from app.core.openai.requests import ResponsesRequest +from app.modules.proxy import http_bridge_forwarding as forwarding + + +@pytest.fixture(autouse=True) +def _bridge_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: + monkeypatch.setenv("CODEX_LB_ENCRYPTION_KEY_FILE", str(tmp_path / "bridge.key")) + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _context() -> forwarding.HTTPBridgeForwardContext: + return forwarding.HTTPBridgeForwardContext( + origin_instance="origin", + target_instance="owner", + codex_session_affinity=False, + downstream_turn_state=None, + client_ip="192.0.2.1", + expected_owner_process_epoch="proven-process", + ) + + +def _payload() -> ResponsesRequest: + return ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "x" * 4095}, + ) + + +@pytest.mark.parametrize( + ("text_length", "requires_upgrade"), + [(4091, False), (4092, True), (4093, True), (4094, False)], +) +def test_one_item_array_compatibility_boundary(text_length: int, requires_upgrade: bool) -> None: + payload = ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": ["x" * text_length]}, + ) + assert forwarding._http_bridge_owner_forward_requires_shape_upgrade(payload) is requires_upgrade + + +@pytest.mark.asyncio +@pytest.mark.parametrize("replacement_epoch", [False, True], ids=["same-process", "replacement-process"]) +@pytest.mark.parametrize( + "input_value", + [ + "x" * 4095, + ["x" * 4092], + ["x" * 4093], + [{"role": "system", "content": "x" * 4096}], + [{"role": "developer", "content": "x" * 4096}], + [{"role": "system", "content": "a"}, {"role": "developer", "content": "b"}], + [ + {"type": "function_call_output", "call_id": "call-1", "output": "first"}, + {"type": "function_call_output", "call_id": "call-2", "output": "second"}, + ], + ], + ids=[ + "raw-string", + "array-boundary", + "array-boundary-upper", + "large-system", + "large-developer", + "multi-instructions", + "parallel-tool-outputs", + ], +) +async def test_owner_forward_checks_proven_epoch_at_http_receive( + monkeypatch: pytest.MonkeyPatch, + replacement_epoch: bool, + input_value: object, +) -> None: + accepted = False + rejected = False + received = False + payload = ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": input_value}, + ) + context = _context() + # Capability proof was for this epoch, but a new process can occupy the + # same endpoint by the time the HTTP request arrives. + monkeypatch.setattr( + forwarding, + "http_bridge_owner_process_epoch", + lambda: "replacement-process" if replacement_epoch else "proven-process", + ) + + async def receive(request: web.Request) -> web.Response: + nonlocal accepted, received + received = True + assert request.headers[forwarding.HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER] == "proven-process" + assert forwarding.HTTP_BRIDGE_SIGNATURE_HEADER not in request.headers + assert forwarding.HTTP_BRIDGE_CLIENT_IP_SIGNATURE_HEADER not in request.headers + assert forwarding.HTTP_BRIDGE_SIGNATURE_V2_HEADER not in request.headers + assert forwarding.HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER in request.headers + wire_payload = ResponsesRequest.model_validate(await request.json()) + forwarded, error = forwarding.parse_forwarded_request( + request.headers, + payload=wire_payload, + current_instance="owner", + ) + if error is not None: + return web.json_response(error.payload, status=error.status_code) + assert forwarded is not None + assert not wire_payload._codex_lb_legacy_owner_forwarding_input_shape + assert forwarding._http_bridge_payload_looks_like_full_resend(wire_payload) == ( + forwarding._http_bridge_payload_looks_like_full_resend(payload) + ) + accepted = True + return web.Response( + text='data: {"type":"response.completed","response":{"id":"resp-test","status":"completed"}}\n\n', + content_type="text/event-stream", + ) + + def response_rejected() -> None: + nonlocal rejected + rejected = True + + app = web.Application() + app.router.add_post(forwarding.HTTP_BRIDGE_INTERNAL_FORWARD_PATH, receive) + async with TestServer(app) as server: + stream = forwarding.HTTPBridgeOwnerClient().stream_responses( + owner_endpoint=str(server.make_url("")).rstrip("/"), + payload=payload, + headers={}, + context=context, + request_started_at=time.monotonic(), + owner_supports_input_shape_classifier=True, + on_response_rejected=response_rejected, + ) + if replacement_epoch: + with pytest.raises(ProxyResponseError) as exc_info: + _ = [event async for event in stream] + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "bridge_owner_forward_failed" + else: + events = [event async for event in stream] + assert len(events) == 1 + assert "response.completed" in events[0] + assert received + assert accepted is not replacement_epoch + assert rejected is replacement_epoch + + +@pytest.mark.parametrize( + "tamper", + ["strip-epoch", "change-epoch", "strip-shape-marker", "strip-exact-signature", "primary-only"], +) +def test_epoch_bound_forward_cannot_downgrade( + monkeypatch: pytest.MonkeyPatch, + tamper: str, +) -> None: + payload = _payload() + context = _context() + headers = forwarding.build_owner_forward_headers(headers={}, payload=payload, context=context) + monkeypatch.setattr(forwarding, "http_bridge_owner_process_epoch", lambda: "proven-process") + if tamper == "strip-epoch": + headers.pop(forwarding.HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER) + elif tamper == "change-epoch": + headers[forwarding.HTTP_BRIDGE_OWNER_PROCESS_EPOCH_HEADER] = "different-process" + elif tamper == "strip-shape-marker": + headers.pop(forwarding.HTTP_BRIDGE_INPUT_SHAPE_VERSION_HEADER) + else: + headers.pop(forwarding.HTTP_BRIDGE_INPUT_SHAPE_SIGNATURE_HEADER) + if tamper == "primary-only": + headers[forwarding.HTTP_BRIDGE_SIGNATURE_HEADER] = forwarding._bridge_forward_signature( + payload=payload, + context=context, + ) + forwarded, error = forwarding.parse_forwarded_request(headers, payload=payload, current_instance="owner") + assert forwarded is None + assert error is not None + assert error.status_code == 400 + + +@pytest.mark.asyncio +async def test_capability_boolean_without_epoch_does_not_authorize_dispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def unexpected_session(**_kwargs: object) -> None: + pytest.fail("A capability boolean without a signed epoch must not dispatch") + + monkeypatch.setattr(forwarding.aiohttp, "ClientSession", unexpected_session) + context = forwarding.HTTPBridgeForwardContext( + origin_instance="origin", + target_instance="owner", + codex_session_affinity=False, + downstream_turn_state=None, + ) + with pytest.raises(ProxyResponseError) as exc_info: + _ = [ + event + async for event in forwarding.HTTPBridgeOwnerClient().stream_responses( + owner_endpoint="http://owner", + payload=_payload(), + headers={}, + context=context, + request_started_at=time.monotonic(), + owner_supports_input_shape_classifier=True, + ) + ] + assert exc_info.value.status_code == 503 + assert exc_info.value.failure_detail == "owner_input_shape_upgrade_required" + + +@pytest.mark.parametrize( + ("input_value", "requires_upgrade"), + [ + ([], False), + ([{"role": "system", "content": "short"}], False), + ([{"role": "developer", "content": "short"}], False), + ([{"role": "system", "content": "x" * 4096}], True), + ([{"role": "developer", "content": "x" * 4096}], True), + ([{"role": "system", "content": "a"}, {"role": "developer", "content": "b"}], True), + ], + ids=["empty", "small-system", "small-developer", "large-system", "large-developer", "multi-instructions"], +) +def test_normalized_empty_classifier_disagreement(input_value: object, requires_upgrade: bool) -> None: + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": input_value}) + assert payload.input == [] + assert forwarding._http_bridge_owner_forward_requires_shape_upgrade(payload) is requires_upgrade + + +@pytest.mark.asyncio +@pytest.mark.parametrize("has_capability", [False, True], ids=["unsupported-owner", "missing-epoch"]) +@pytest.mark.parametrize( + "input_value", + [ + [{"role": "system", "content": "x" * 4096}], + [{"role": "developer", "content": "x" * 4096}], + [{"role": "system", "content": "a"}, {"role": "developer", "content": "b"}], + ], + ids=["large-system", "large-developer", "multi-instructions"], +) +async def test_normalized_empty_disagreement_rejected_before_dispatch( + monkeypatch: pytest.MonkeyPatch, has_capability: bool, input_value: object +) -> None: + def unexpected_session(**_kwargs: object) -> None: + pytest.fail("Classifier disagreement must fail before HTTP session creation") + + def unexpected_dispatch() -> None: + pytest.fail("Rejected classifier disagreement must not mark the request dispatched") + + monkeypatch.setattr(forwarding.aiohttp, "ClientSession", unexpected_session) + context = replace(_context(), expected_owner_process_epoch=None) if has_capability else _context() + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": input_value}) + assert payload.input == [] + with pytest.raises(ProxyResponseError) as exc_info: + _ = [ + event + async for event in forwarding.HTTPBridgeOwnerClient().stream_responses( + owner_endpoint="http://owner", + payload=payload, + headers={}, + context=context, + request_started_at=time.monotonic(), + owner_supports_input_shape_classifier=has_capability, + on_request_dispatched=unexpected_dispatch, + ) + ] + assert exc_info.value.status_code == 503 + assert exc_info.value.failure_detail == "owner_input_shape_upgrade_required" diff --git a/tests/unit/test_http_bridge_forwarding_settings.py b/tests/unit/test_http_bridge_forwarding_settings.py new file mode 100644 index 0000000000..693fda864d --- /dev/null +++ b/tests/unit/test_http_bridge_forwarding_settings.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from typing import NoReturn + +import aiohttp +import pytest + +from app.core.clients.proxy import ProxyResponseError +from app.core.config.dashboard_overrides import dashboard_overrides_bound +from app.core.config.settings import Settings +from app.core.openai.requests import ResponsesRequest +from app.db.models import DashboardSettings +from app.modules.proxy import http_bridge_forwarding as forwarding + + +class _ReachedOwnerSession(Exception): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capability_proven", [False, True]) +@pytest.mark.parametrize("dashboard_timeout", [None, 3.25]) +async def test_dashboard_timeouts_preserve_owner_classifier_gate( + monkeypatch: pytest.MonkeyPatch, + capability_proven: bool, + dashboard_timeout: float | None, +) -> None: + base = Settings().model_copy(update={"upstream_connect_timeout_seconds": 10.0, "stream_idle_timeout_seconds": 60.0}) + monkeypatch.setattr(forwarding, "get_settings", lambda: base) + row = DashboardSettings( + upstream_connect_timeout_seconds=dashboard_timeout, + stream_idle_timeout_seconds=dashboard_timeout, + ) + captured: list[aiohttp.ClientTimeout] = [] + + def capture_session(*, timeout: aiohttp.ClientTimeout, trust_env: bool) -> NoReturn: + assert trust_env is False + captured.append(timeout) + raise _ReachedOwnerSession + + monkeypatch.setattr(forwarding.aiohttp, "ClientSession", capture_session) + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "x" * 4095}) + context = forwarding.HTTPBridgeForwardContext( + origin_instance="origin", + target_instance="owner", + codex_session_affinity=False, + downstream_turn_state=None, + expected_owner_process_epoch="proven-process", + ) + with dashboard_overrides_bound(row): + stream = forwarding.HTTPBridgeOwnerClient().stream_responses( + owner_endpoint="http://owner.invalid", + payload=payload, + headers={}, + context=context, + request_started_at=0.0, + owner_supports_input_shape_classifier=capability_proven, + ) + if capability_proven: + with pytest.raises(_ReachedOwnerSession): + _ = [event async for event in stream] + else: + with pytest.raises(ProxyResponseError) as exc_info: + _ = [event async for event in stream] + assert exc_info.value.failure_phase == "owner_forward" + assert exc_info.value.failure_detail == "owner_input_shape_upgrade_required" + + if capability_proven: + assert len(captured) == 1 + assert captured[0].sock_connect == (dashboard_timeout if dashboard_timeout is not None else 10.0) + assert captured[0].sock_read == (dashboard_timeout if dashboard_timeout is not None else 60.0) + else: + assert captured == [] + assert base.upstream_connect_timeout_seconds == 10.0 + assert base.stream_idle_timeout_seconds == 60.0 diff --git a/tests/unit/test_http_bridge_turn_state_recovery.py b/tests/unit/test_http_bridge_turn_state_recovery.py new file mode 100644 index 0000000000..f8c72644cf --- /dev/null +++ b/tests/unit/test_http_bridge_turn_state_recovery.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import pytest + +from app.core.clients.proxy import ProxyResponseError +from app.modules.proxy._service.http_bridge.helpers import _http_bridge_turn_state_anchor_for_owner_failure + + +@pytest.mark.parametrize( + ("phase", "detail", "code", "eligible"), + [ + ("owner_forward", "owner_input_shape_upgrade_required", "bridge_owner_forward_failed", True), + (None, "owner_input_shape_upgrade_required", "bridge_owner_forward_failed", False), + ("owner_forward", "other", "bridge_owner_forward_failed", False), + (None, None, "bridge_owner_unreachable", True), + (None, None, "bridge_owner_forward_failed", False), + ], +) +def test_turn_state_recovery_requires_unreachable_owner_or_typed_shape_failure( + phase: str | None, detail: str | None, code: str, eligible: bool +) -> None: + error = ProxyResponseError( + 503, + {"error": {"code": code}}, + failure_phase=phase, + failure_detail=detail, + ) + anchor = _http_bridge_turn_state_anchor_for_owner_failure( + error, headers={"X-Codex-Turn-State": "turn-state"}, previous_response_id=None + ) + assert anchor == ("turn-state" if eligible else None) + + +@pytest.mark.parametrize(("turn_state", "previous_response_id"), [(None, None), ("turn-state", "resp-client")]) +def test_shape_failure_does_not_replace_missing_turn_state_or_explicit_response( + turn_state: str | None, previous_response_id: str | None +) -> None: + error = ProxyResponseError( + 503, + {"error": {"code": "bridge_owner_forward_failed"}}, + failure_phase="owner_forward", + failure_detail="owner_input_shape_upgrade_required", + ) + assert ( + _http_bridge_turn_state_anchor_for_owner_failure( + error, + headers={"x-codex-turn-state": turn_state} if turn_state is not None else {}, + previous_response_id=previous_response_id, + ) + is None + ) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index b5c5fe8f84..9ed8928271 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -53,6 +53,7 @@ from app.modules.proxy import affinity as proxy_affinity from app.modules.proxy import api as proxy_api from app.modules.proxy import http_bridge_forwarding as http_bridge_forwarding_module +from app.modules.proxy import ring_membership as ring_membership_module from app.modules.proxy import service as proxy_service from app.modules.proxy._load_balancer.tunables import RoutingTunables from app.modules.proxy._service import support as proxy_support_module @@ -334,6 +335,141 @@ def test_http_bridge_prepares_full_resend_shape_for_late_hard_anchor_injection() assert request_state.proxy_injected_anchor_had_full_resend_payload is True +@pytest.mark.parametrize( + ("input_value", "expected"), + [ + pytest.param("x" * 4095, False, id="short-string"), + pytest.param("x" * 4096, True, id="boundary-string"), + pytest.param(["x" * 4091], False, id="short-one-item-array"), + pytest.param(["x" * 4092], True, id="boundary-one-item-array"), + pytest.param(["x", "y"], True, id="multiple-items"), + pytest.param( + [ + {"type": "function_call_output", "call_id": "call-1", "output": "first"}, + {"type": "function_call_output", "call_id": "call-2", "output": "second"}, + ], + False, + id="parallel-tool-output-delta", + ), + pytest.param([], False, id="empty-array"), + ], +) +def test_http_bridge_full_resend_shape_classifier_has_normative_boundaries( + input_value: proxy_service.JsonValue, + expected: bool, +) -> None: + payload = ResponsesRequest.model_construct(model="gpt-5.6", instructions="", input=input_value) + + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is expected + + +@pytest.mark.parametrize( + ("input_length", "expected"), + [ + pytest.param(4035, False, id="normalized-array-overhead-does-not-count"), + pytest.param(4095, False, id="raw-string-below-boundary"), + pytest.param(4096, True, id="raw-string-at-boundary"), + ], +) +def test_http_bridge_full_resend_shape_preserves_validated_raw_string_boundary( + input_length: int, + expected: bool, +) -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6", + "instructions": "", + "input": "x" * input_length, + } + ) + + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is expected + revalidated = ResponsesRequest.model_validate(payload.model_dump_for_http_bridge_owner_forwarding()) + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(revalidated) is expected + + +def test_http_bridge_full_resend_shape_preserves_raw_multi_item_array_after_instruction_hoisting() -> None: + raw_input = [ + {"role": "developer", "content": "developer instructions"}, + {"role": "user", "content": "follow-up"}, + ] + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6", + "instructions": "", + "input": raw_input, + } + ) + + assert payload.instructions == "developer instructions" + assert payload.input == [{"role": "user", "content": "follow-up"}] + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is True + + owner_wire = payload.model_dump_for_http_bridge_owner_forwarding() + assert owner_wire["input"] == raw_input + assert owner_wire["instructions"] == "" + owner_payload = ResponsesRequest.model_validate(owner_wire) + assert owner_payload.instructions == "developer instructions" + assert owner_payload.input == [{"role": "user", "content": "follow-up"}] + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(owner_payload) is True + + +def test_http_bridge_legacy_one_item_large_array_remains_full_resend() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6", + "instructions": "", + "input": ["x" * 4094], + } + ) + payload._codex_lb_legacy_owner_forwarding_input_shape = True + + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(payload) is True + + +def test_http_bridge_full_resend_shape_does_not_reuse_raw_string_length_after_input_replacement() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6", + "instructions": "", + "input": "x" * 4096, + } + ) + replaced = payload.model_copy(update={"input": ["delta"]}) + + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(replaced) is False + assert replaced.model_dump_for_http_bridge_owner_forwarding()["input"] == ["delta"] + + +@pytest.mark.parametrize( + ("input_length", "expected"), + [ + pytest.param(4035, False, id="normalized-array-overhead-does-not-count"), + pytest.param(4095, False, id="raw-string-below-boundary"), + pytest.param(4096, True, id="raw-string-at-boundary"), + ], +) +def test_http_bridge_owner_forwarding_preserves_raw_string_boundary( + input_length: int, + expected: bool, +) -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6", + "instructions": "", + "input": "x" * input_length, + } + ) + + forwarded = payload.model_dump_for_http_bridge_owner_forwarding() + assert isinstance(forwarded["input"], str) + assert isinstance(payload.model_dump_for_forwarding()["input"], list) + owner_payload = ResponsesRequest.model_validate(forwarded) + + assert http_bridge_helpers_module._http_bridge_payload_looks_like_full_resend(owner_payload) is expected + + def test_http_bridge_operation_fingerprint_strips_account_installation_metadata() -> None: request = ( '{"type":"response.create","previous_response_id":"resp_parent",' @@ -737,6 +873,75 @@ def test_http_bridge_explicit_previous_response_rejection_normalizes_error_type( assert proxy_service._http_bridge_is_explicit_previous_response_rejection(ProxyResponseError(400, error)) is True +def test_http_bridge_owner_input_shape_upgrade_failure_allows_local_recovery() -> None: + error = proxy_service.openai_error( + "bridge_owner_forward_failed", + "HTTP bridge owner cannot safely classify this continuation", + ) + exc = ProxyResponseError( + 503, + error, + failure_phase="owner_forward", + failure_detail="owner_input_shape_upgrade_required", + ) + + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(exc) is True + assert ( + proxy_service._http_bridge_should_attempt_local_bootstrap_rebind( + exc, + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-shape-upgrade", None), + headers={"x-codex-session-id": "sid-shape-upgrade"}, + previous_response_id=None, + ) + is True + ) + + +def test_http_bridge_owner_input_shape_upgrade_failure_preserves_affinity_scope() -> None: + error = proxy_service.openai_error( + "bridge_owner_forward_failed", + "HTTP bridge owner cannot safely classify this continuation", + ) + exc = ProxyResponseError( + 503, + error, + failure_phase="owner_forward", + failure_detail="owner_input_shape_upgrade_required", + ) + + for key, headers, previous_response_id in ( + ( + proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn-state", None), + {"x-codex-turn-state": "http_turn_123"}, + None, + ), + ( + proxy_service._HTTPBridgeSessionKey("internal_request_parallel", "request-parallel", None), + {}, + None, + ), + ( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-shape-upgrade", None), + {"x-codex-session-id": "sid-shape-upgrade", "x-codex-turn-state": "http_turn_123"}, + None, + ), + ( + proxy_service._HTTPBridgeSessionKey("session_header", "sid-shape-upgrade", None), + {"x-codex-session-id": "sid-shape-upgrade"}, + "resp-prev-1", + ), + ): + assert ( + proxy_service._http_bridge_should_attempt_local_bootstrap_rebind( + exc, + key=key, + headers=headers, + previous_response_id=previous_response_id, + ) + is False + ) + + @pytest.mark.parametrize("code", ["previous_response_not_found", "bridge_previous_response_not_found"]) @pytest.mark.parametrize("param", [None, "", " ", 0, False, {}, []]) def test_http_bridge_stale_anchor_recovery_rejects_malformed_present_param(code: str, param: object) -> None: @@ -14764,19 +14969,45 @@ async def test_stream_via_http_bridge_skips_session_anchor_after_cross_account_f @pytest.mark.asyncio -async def test_stream_via_http_bridge_does_not_inject_durable_previous_response_anchor_for_full_resend_payload( +@pytest.mark.parametrize( + ("input_value", "quarantine_key", "expected_anchor", "expected_prepare_lengths"), + [ + pytest.param( + [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "world"}, + {"role": "user", "content": "follow up"}, + ], + False, + None, + [3], + id="full-resend-stays-unanchored", + ), + pytest.param( + [ + {"type": "function_call_output", "call_id": "call-1", "output": "first"}, + {"type": "function_call_output", "call_id": "call-2", "output": "second"}, + ], + True, + "resp_latest", + [2, 2], + id="quarantined-parallel-tool-output-delta-keeps-anchor", + ), + ], +) +async def test_stream_via_http_bridge_classifies_anchorless_full_resend_and_quarantined_multi_output_delta( monkeypatch: pytest.MonkeyPatch, + input_value: list[proxy_service.JsonValue], + quarantine_key: bool, + expected_anchor: str | None, + expected_prepare_lengths: list[int], ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.4", "instructions": "hi", - "input": [ - {"role": "user", "content": "hello"}, - {"role": "assistant", "content": "world"}, - {"role": "user", "content": "follow up"}, - ], + "input": input_value, }, ) request_state = proxy_service._WebSocketRequestState( @@ -14828,6 +15059,12 @@ def fake_prepare( last_used_at=1.0, idle_ttl_seconds=120.0, ) + if quarantine_key: + http_bridge_quarantine_module._quarantine_http_bridge_session( + service, + _make_bridge_session(key_value="sid-123"), + reason=http_bridge_quarantine_module._HTTP_BRIDGE_QUARANTINE_WEDGED_REATTACH_REASON, + ) monkeypatch.setattr( proxy_service, @@ -14890,15 +15127,83 @@ def fake_prepare( ] assert chunks == [] - assert captured["previous_response_id"] is None - # Full-resend payloads are explicitly excluded from durable anchor - # injection, so the bridge prepares the original request exactly once. - assert prepared_input_lengths == [3] - # This path never reaches the trim branch, so the fake request_state - # returned by fake_prepare keeps its default metadata. + assert captured["previous_response_id"] == expected_anchor + assert prepared_input_lengths == expected_prepare_lengths assert request_state.input_full_fingerprint is None +@pytest.mark.asyncio +async def test_stream_via_http_bridge_classifies_full_resend_before_continuity_lookup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The payload-shape guard runs before legacy or durable continuity I/O.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "", + "input": [ + {"role": "user", "content": "first"}, + {"role": "user", "content": "second"}, + ], + } + ) + events: list[str] = [] + + def classify(_payload: proxy_service.ResponsesRequest) -> bool: + events.append("classify") + return True + + async def legacy_lookup(**_kwargs: Any) -> None: + events.append("legacy") + return None + + async def durable_lookup(**_kwargs: Any) -> None: + events.append("durable") + raise ProxyResponseError(502, openai_error("lookup_failed", "stop after ordering assertion")) + + async def settings_get() -> SimpleNamespace: + events.append("settings") + return SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_payload_looks_like_full_resend", classify) + monkeypatch.setattr(http_bridge_streaming_module, "_legacy_forward_anchor_lookup", legacy_lookup) + service._durable_bridge = cast(Any, SimpleNamespace(lookup_request_targets=durable_lookup)) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace(get=settings_get), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + + with pytest.raises(ProxyResponseError, match="Proxy response error"): + async for _chunk in service._stream_via_http_bridge_impl( + payload, + headers={}, + codex_session_affinity=False, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ): + pass + + assert events == ["classify", "settings", "legacy", "durable"] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("suffix_items", "pending_tool_calls", "preserves_full_resend", "forwardable_owner"), @@ -17792,6 +18097,68 @@ async def fake_stream_responses(**kwargs: object): assert cast(dict[str, str], captured["headers"])["x-codex-session-id"] == "sid-123" +@pytest.mark.asyncio +async def test_forward_http_bridge_request_to_owner_proves_ambiguous_shape_capability( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = VirtualClock(monotonic_value=10.0) + scheduler = VirtualScheduler(clock) + service = proxy_service.ProxyService(cast(Any, nullcontext()), scheduler=scheduler, clock=clock) + owner_forward = proxy_service._HTTPBridgeOwnerForward( + owner_instance="instance-b", + owner_endpoint="http://instance-b", + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), + owner_process_epoch="owner-process-b", + ) + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": "x" * 4095}, + ) + capability_check = AsyncMock(return_value=True) + service._ring_membership = cast( + Any, + SimpleNamespace(owner_supports_capability=capability_check), + ) + captured: dict[str, object] = {} + + async def fake_stream_responses(**kwargs: object): + captured.update(kwargs) + if False: + yield "" + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service, + "_http_bridge_owner_client", + SimpleNamespace(stream_responses=fake_stream_responses), + ) + + chunks = [ + chunk + async for chunk in service._forward_http_bridge_request_to_owner( + owner_forward=owner_forward, + payload=payload, + headers={}, + api_key_reservation=None, + codex_session_affinity=True, + downstream_turn_state="http_turn_generated", + request_started_at=10.0, + proxy_api_authorization=None, + ) + ] + + assert chunks == [] + capability_check.assert_awaited_once_with( + "instance-b", + owner_process_epoch="owner-process-b", + capability=ring_membership_module.HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, + ) + assert captured["owner_supports_input_shape_classifier"] is True + assert captured["clock"] is clock + assert captured["scheduler"] is scheduler + context = cast(proxy_service.HTTPBridgeForwardContext, captured["context"]) + assert context.expected_owner_process_epoch == "owner-process-b" + + @pytest.mark.asyncio async def test_recovery_forward_replaces_incoming_affinity_with_recovered_turn_state( monkeypatch: pytest.MonkeyPatch, @@ -19398,6 +19765,232 @@ async def test_stream_via_http_bridge_owner_forward_recovery_without_pending_sta assert prepared_inputs == [input_items, input_items] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("continuation", "injected_anchor"), + [ + ("previous-response", False), + ("turn-state", False), + ("turn-state-live-lease", False), + ("turn-state-lookup-error", False), + ("turn-state", True), + ("turn-state-live-lease", True), + ("turn-state-lookup-error", True), + ("turn-state-other-account", True), + ], +) +async def test_stream_via_http_bridge_owner_input_shape_upgrade_recovers_locally_without_owner_process_epoch( + monkeypatch: pytest.MonkeyPatch, + continuation: str, + injected_anchor: bool, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + previous_response_id = "resp_prev_1" if continuation == "previous-response" else None + input_items = [ + {"type": "function_call_output", "call_id": "call-1", "output": "one"}, + {"type": "function_call_output", "call_id": "call-2", "output": "two"}, + ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": input_items, + "previous_response_id": previous_response_id, + } + ) + started_at = time.monotonic() + prepared_inputs: list[Any] = [] + prepared_anchors: list[str | None] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + prepared_anchors.append(prepared_payload.previous_response_id) + prepared_inputs.append(prepared_payload.input) + state = proxy_service._WebSocketRequestState( + request_id=f"req-{len(prepared_inputs)}", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=started_at, + event_queue=asyncio.Queue(), + transport="http", + previous_response_id=prepared_payload.previous_response_id, + ) + return state, '{"type":"response.create"}' + + owner_forward = proxy_service._HTTPBridgeOwnerForward( + owner_instance="instance-b", + owner_endpoint="http://instance-b", + key=proxy_service._HTTPBridgeSessionKey( + "session_header" if previous_response_id is not None else "turn_state_header", + "sid-shape-upgrade" if previous_response_id is not None else "turn-shape-upgrade", + None, + ), + ) + recovery_session = _make_owner_forward_recovery_session() + capability_probe = AsyncMock() + service._ring_membership = cast( + Any, + SimpleNamespace(owner_supports_capability=capability_probe), + ) + + async def fake_submit_http_bridge_request( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del _session, text_data, queue_limit + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put('data: {"type":"response.completed"}\n\n') + await event_queue.put(None) + + get_or_create = AsyncMock(side_effect=[owner_forward, recovery_session]) + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + initial_lookup = ( + proxy_service.DurableBridgeLookup( + session_id="durable-shape-upgrade", + canonical_kind="turn_state_header", + canonical_key="turn-shape-upgrade", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=None, + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="turn-shape-upgrade", + latest_response_id="resp-durable-shape", + ) + if injected_anchor + else None + ) + fresh_lookup: proxy_service.DurableBridgeLookup | RuntimeError | None = initial_lookup + if continuation == "turn-state-live-lease": + fresh_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-shape-upgrade", + canonical_kind="turn_state_header", + canonical_key="turn-shape-upgrade", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-b", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="turn-shape-upgrade", + latest_response_id=None, + ) + elif continuation == "turn-state-lookup-error": + fresh_lookup = RuntimeError("durable lookup unavailable") + elif continuation == "turn-state-other-account": + assert initial_lookup is not None + fresh_lookup = replace(initial_lookup, account_id="acc-other", latest_response_id="resp-other") + lookup_targets = AsyncMock(side_effect=[initial_lookup, fresh_lookup]) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", lookup_targets) + monkeypatch.setattr(service._durable_bridge, "lookup_turn_state_target", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "lookup_retry_circuit", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + def unexpected_owner_io(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("owner I/O must not start before shape capability is proven") + + monkeypatch.setattr( + http_bridge_forwarding_module.aiohttp, + "ClientSession", + unexpected_owner_io, + ) + + headers = ( + {"x-codex-session-id": "sid-shape-upgrade"} + if previous_response_id is not None + else {"x-codex-turn-state": "turn-shape-upgrade"} + ) + stream = service._stream_via_http_bridge( + payload, + headers=headers, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + max_sessions=8, + queue_limit=4, + ) + initial_prepares = 2 if injected_anchor else 1 + if continuation in {"turn-state-live-lease", "turn-state-lookup-error", "turn-state-other-account"}: + with pytest.raises(ProxyResponseError) as exc_info: + _ = [chunk async for chunk in stream] + if continuation != "turn-state-other-account": + assert exc_info.value.failure_phase == "owner_forward" + assert exc_info.value.failure_detail == "owner_input_shape_upgrade_required" + else: + assert exc_info.value.payload["error"]["code"] == "continuity_owner_conflict" + assert prepared_inputs == [input_items] * initial_prepares + assert prepared_anchors == ([None, "resp-durable-shape"] if injected_anchor else [previous_response_id]) + assert get_or_create.await_count == 1 + assert lookup_targets.await_count == 2 + capability_probe.assert_not_awaited() + return + chunks = [chunk async for chunk in stream] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + expected_anchor = "resp-durable-shape" if injected_anchor else previous_response_id + assert prepared_inputs == [input_items] * (initial_prepares + 1) + assert prepared_anchors == ([None, expected_anchor, expected_anchor] if injected_anchor else [expected_anchor] * 2) + capability_probe.assert_not_awaited() + assert get_or_create.await_count == 2 + recovery_kwargs = get_or_create.await_args_list[1].kwargs + assert recovery_kwargs["allow_forward_to_owner"] is False + assert recovery_kwargs["allow_previous_response_recovery_rebind"] is (previous_response_id is not None) + assert recovery_kwargs["allow_bootstrap_owner_rebind"] is (previous_response_id is None) + assert recovery_kwargs["previous_response_id"] == expected_anchor + assert recovery_kwargs["request_stage"] == "reattach" + if previous_response_id is None: + assert lookup_targets.await_count == 2 + assert lookup_targets.await_args is not None + assert lookup_targets.await_args.kwargs["turn_state"] == "turn-shape-upgrade" + assert lookup_targets.await_args.kwargs["previous_response_id"] is None + if injected_anchor: + assert recovery_kwargs["preferred_account_id"] == "acc-1" + + async def _run_owner_forward_recovery_durable_anchor_stream( monkeypatch: pytest.MonkeyPatch, *, @@ -28682,11 +29275,26 @@ async def test_get_or_create_http_bridge_session_sticky_thread_mismatch_forwards max_sessions=8, allow_forward_to_owner=True, gateway_safe_mode=True, + durable_lookup=proxy_service.DurableBridgeLookup( + session_id="durable-sticky-thread", + canonical_kind="sticky_thread", + canonical_key="thread-key", + api_key_scope="__anonymous__", + account_id="acc-owner", + owner_instance_id="instance-b", + owner_process_epoch="owner-process-b", + owner_epoch=2, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=None, + ), ) assert isinstance(resolved, proxy_service._HTTPBridgeOwnerForward) assert resolved.owner_instance == "instance-b" assert resolved.owner_endpoint == "http://instance-b" + assert resolved.owner_process_epoch == "owner-process-b" @pytest.mark.asyncio diff --git a/tests/unit/test_ring_membership.py b/tests/unit/test_ring_membership.py index 280bcabc0b..033f417f01 100644 --- a/tests/unit/test_ring_membership.py +++ b/tests/unit/test_ring_membership.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import AsyncIterator, Callable from datetime import timedelta @@ -8,7 +9,9 @@ from app.core.utils.time import utcnow from app.db.models import Base, BridgeRingMember +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.ring_membership import ( + HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, RING_HEARTBEAT_INTERVAL_SECONDS, RING_STALE_GRACE_SECONDS, RING_STALE_THRESHOLD_SECONDS, @@ -181,6 +184,53 @@ async def test_resolve_endpoint_returns_advertised_base_url(ring_service: RingMe assert endpoint == "http://10.0.0.12:8080" +@pytest.mark.asyncio +async def test_owner_capability_proof_is_bound_to_current_process_epoch( + ring_service: RingMembershipService, +) -> None: + await ring_service.register("pod-capable", endpoint_base_url="http://10.0.0.13:8080") + + assert await ring_service.owner_supports_capability( + "pod-capable", + owner_process_epoch=http_bridge_owner_process_epoch(), + capability=HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, + ) + assert not await ring_service.owner_supports_capability( + "pod-capable", + owner_process_epoch="stale-process-epoch", + capability=HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, + ) + + +@pytest.mark.asyncio +async def test_legacy_owner_advertisement_does_not_prove_current_classifier( + ring_service: RingMembershipService, +) -> None: + await ring_service.register("pod-legacy", endpoint_base_url="http://10.0.0.14:8080") + async with ring_service._session() as session: + from sqlalchemy import update + + await session.execute( + update(BridgeRingMember) + .where(BridgeRingMember.instance_id == "pod-legacy") + .values( + metadata_json=json.dumps( + { + "endpoint_base_url": "http://10.0.0.14:8080", + "owner_process_epoch": http_bridge_owner_process_epoch(), + } + ) + ) + ) + await session.commit() + + assert not await ring_service.owner_supports_capability( + "pod-legacy", + owner_process_epoch=http_bridge_owner_process_epoch(), + capability=HTTP_BRIDGE_INPUT_SHAPE_CLASSIFIER_CAPABILITY, + ) + + @pytest.mark.asyncio async def test_resolve_endpoint_ignores_stale_member_metadata(ring_service: RingMembershipService) -> None: await ring_service.register("pod-stale-endpoint", endpoint_base_url="http://10.0.0.14:8080")