From 52ccae4740d355dc035714f06232f505ae045edc Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:01:43 -0700 Subject: [PATCH] refactor: move the Codex recovery rungs out of turn_recovery turn_recovery.py crossed the 2,000-line gate once the reasoning-replay recovery landed beside it. The stale-encrypted-reasoning rungs and the replay-verdict reset are one topic; they move verbatim to agent/turn_recovery_codex.py. --- agent/agent_runtime_helpers.py | 2 +- agent/route_binding.py | 4 +- agent/turn_recovery.py | 66 +---------------------------- agent/turn_recovery_codex.py | 76 ++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 68 deletions(-) create mode 100644 agent/turn_recovery_codex.py diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 619e01340e..15cd862855 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -2286,7 +2286,7 @@ def _swap_switch_runtime(agent, new_model, new_provider, api_key, base_url, api_ # New api_mode may need a different transport. if hasattr(agent, "_transport_cache"): agent._transport_cache.clear() - from agent.turn_recovery import reset_codex_reasoning_replay + from agent.turn_recovery_codex import reset_codex_reasoning_replay reset_codex_reasoning_replay(agent) if api_key: agent.api_key = api_key diff --git a/agent/route_binding.py b/agent/route_binding.py index 5a674612c3..7981c329d3 100644 --- a/agent/route_binding.py +++ b/agent/route_binding.py @@ -69,7 +69,7 @@ def bind_route_entry(agent: Any, entry: dict[str, Any], provider: str, model: st agent._reasoning_echo_flag = bool(entry.get("reasoning_echo", False)) if hasattr(agent, "_transport_cache"): agent._transport_cache.clear() - from agent.turn_recovery import reset_codex_reasoning_replay + from agent.turn_recovery_codex import reset_codex_reasoning_replay reset_codex_reasoning_replay(agent) agent._fallback_activated = True @@ -101,7 +101,7 @@ def reinstall_runtime_snapshot(agent: Any, rt: dict[str, Any]) -> None: _apply_primary_runtime_fields, _rebuild_primary_client, _restore_runtime_capabilities, ) _apply_primary_runtime_fields(agent, rt) - from agent.turn_recovery import reset_codex_reasoning_replay + from agent.turn_recovery_codex import reset_codex_reasoning_replay reset_codex_reasoning_replay(agent) _restore_runtime_capabilities(agent, rt) agent._use_prompt_caching = rt["use_prompt_caching"] diff --git a/agent/turn_recovery.py b/agent/turn_recovery.py index 2eb52f1069..635fb55297 100644 --- a/agent/turn_recovery.py +++ b/agent/turn_recovery.py @@ -34,6 +34,7 @@ from agent.turn_failure_copy import ( provider_label_for, site_copy, stamp_failure, ) from agent.turn_retry_state import TurnRetryState +from agent.turn_recovery_codex import _is_codex_token_expired, _recover_stale_codex_reasoning from hermes_constants import display_hermes_home from utils import base_url_host_matches @@ -450,71 +451,6 @@ def _refresh_credentials_after_401( return False -def _is_codex_token_expired(agent: Any, api_error: Exception) -> bool: - """401 ``token_expired`` from the Codex backend (#88510). It rejects a stale replayed - ``encrypted_content`` blob with this auth signature, so a persisted session loops on "sign - in again" while a fresh session on the same bearer works. The caller treats it like - ``invalid_encrypted_content`` — but only while cached reasoning items remain to strip.""" - if getattr(api_error, "status_code", None) != 401: - return False - reason = agent._extract_api_error_context(api_error).get("reason") - return isinstance(reason, str) and reason.strip().lower() == "token_expired" - - -def reset_codex_reasoning_replay(agent: Any) -> None: - """The replay verdict belongs to the route that earned it: a ``/model`` switch, fallback - activation or primary restore starts the new route with replay on (#61552).""" - agent._codex_reasoning_replay_enabled = True - agent._codex_reasoning_replay_rejected = False - - -def _recover_stale_codex_reasoning( - agent: Any, _retry: TurnRetryState, messages: list[dict[str, Any]], api_messages: Any, -) -> bool: - """Stale ``codex_reasoning_items`` blob rejected by the provider: strip cached items (mutates - persisted ``messages``) and retry once. The first rung drops only the replayed blobs not stamped by - exactly this endpoint+model (it consumes its own precondition, so it cannot loop); a repeat goes on - to strip every item. The first full strip keeps replay on, since blobs the - route mints from now on are sealed with its current key; a repeat rejection means the route - cannot round-trip its own blobs, so replay is disabled for the session.""" - if ( - _retry.invalid_encrypted_content_retry_attempted - or agent.api_mode != "codex_responses" - or not bool(getattr(agent, "_codex_reasoning_replay_enabled", True)) - or not any( - isinstance(_m, dict) - and _m.get("role") == "assistant" - and isinstance(_m.get("codex_reasoning_items"), list) - and _m.get("codex_reasoning_items") - for _m in messages - ) - ): - return False - transport = agent._get_transport() - if transport is not None and (removed := transport.drop_unverified_replay(messages, api_messages)): - _vlines(agent, f"⚠️ Encrypted reasoning replay was rejected — dropped {removed} item(s) minted elsewhere, retrying...") - return True - _retry.invalid_encrypted_content_retry_attempted = True - keep_replay = not getattr(agent, "_codex_reasoning_replay_rejected", False) - agent._codex_reasoning_replay_rejected = True - replay_stats = agent._disable_codex_reasoning_replay(messages, keep_replay=keep_replay) - # The retry is rebuilt from the request copy; with replay kept on it would resend the stale blob. - for _m in api_messages if isinstance(api_messages, list) else []: - if isinstance(_m, dict): - _m.pop("codex_reasoning_items", None) - action = "stripped stale" if keep_replay else "disabled replay for this session and stripped" - _vlines( - agent, - f"⚠️ Encrypted reasoning replay was rejected by the provider — " - f"{action} {replay_stats['items']} item(s) from {replay_stats['messages']} message(s), retrying...", - ) - logger.warning( - "%sInvalid encrypted reasoning recovery: %s %d items from %d messages", - agent.log_prefix, action, replay_stats["items"], replay_stats["messages"], - ) - return True - - def _recover_format_errors( agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState, messages: list[dict[str, Any]], api_messages: Any, diff --git a/agent/turn_recovery_codex.py b/agent/turn_recovery_codex.py new file mode 100644 index 0000000000..c507bc76ce --- /dev/null +++ b/agent/turn_recovery_codex.py @@ -0,0 +1,76 @@ +"""Codex Responses recovery rungs: stale encrypted-reasoning strip and the replay verdict reset.""" +from __future__ import annotations + +import logging +from typing import Any + +from agent.turn_retry_state import TurnRetryState + +logger = logging.getLogger(__name__) + + +def _is_codex_token_expired(agent: Any, api_error: Exception) -> bool: + """401 ``token_expired`` from the Codex backend (#88510). It rejects a stale replayed + ``encrypted_content`` blob with this auth signature, so a persisted session loops on "sign + in again" while a fresh session on the same bearer works. The caller treats it like + ``invalid_encrypted_content`` — but only while cached reasoning items remain to strip.""" + if getattr(api_error, "status_code", None) != 401: + return False + reason = agent._extract_api_error_context(api_error).get("reason") + return isinstance(reason, str) and reason.strip().lower() == "token_expired" + + +def reset_codex_reasoning_replay(agent: Any) -> None: + """The replay verdict belongs to the route that earned it: a ``/model`` switch, fallback + activation or primary restore starts the new route with replay on (#61552).""" + agent._codex_reasoning_replay_enabled = True + agent._codex_reasoning_replay_rejected = False + + +def _recover_stale_codex_reasoning( + agent: Any, _retry: TurnRetryState, messages: list[dict[str, Any]], api_messages: Any, +) -> bool: + """Stale ``codex_reasoning_items`` blob rejected by the provider: strip cached items (mutates + persisted ``messages``) and retry once. The first rung drops only the replayed blobs not stamped by + exactly this endpoint+model (it consumes its own precondition, so it cannot loop); a repeat goes on + to strip every item. The first full strip keeps replay on, since blobs the + route mints from now on are sealed with its current key; a repeat rejection means the route + cannot round-trip its own blobs, so replay is disabled for the session.""" + if ( + _retry.invalid_encrypted_content_retry_attempted + or agent.api_mode != "codex_responses" + or not bool(getattr(agent, "_codex_reasoning_replay_enabled", True)) + or not any( + isinstance(_m, dict) + and _m.get("role") == "assistant" + and isinstance(_m.get("codex_reasoning_items"), list) + and _m.get("codex_reasoning_items") + for _m in messages + ) + ): + return False + from agent.turn_recovery import _vlines # facade owns the verbose printer; late import breaks the cycle + + transport = agent._get_transport() + if transport is not None and (removed := transport.drop_unverified_replay(messages, api_messages)): + _vlines(agent, f"⚠️ Encrypted reasoning replay was rejected — dropped {removed} item(s) minted elsewhere, retrying...") + return True + _retry.invalid_encrypted_content_retry_attempted = True + keep_replay = not getattr(agent, "_codex_reasoning_replay_rejected", False) + agent._codex_reasoning_replay_rejected = True + replay_stats = agent._disable_codex_reasoning_replay(messages, keep_replay=keep_replay) + # The retry is rebuilt from the request copy; with replay kept on it would resend the stale blob. + for _m in api_messages if isinstance(api_messages, list) else []: + if isinstance(_m, dict): + _m.pop("codex_reasoning_items", None) + action = "stripped stale" if keep_replay else "disabled replay for this session and stripped" + _vlines( + agent, + f"⚠️ Encrypted reasoning replay was rejected by the provider — " + f"{action} {replay_stats['items']} item(s) from {replay_stats['messages']} message(s), retrying...", + ) + logger.warning( + "%sInvalid encrypted reasoning recovery: %s %d items from %d messages", + agent.log_prefix, action, replay_stats["items"], replay_stats["messages"], + ) + return True