mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
A compute-host interrupt failure returned 5019 and left the session running=True forever: nothing could deliver the turn.end that clears it, so every later prompt.submit fell into the busy path and queued, with no way back short of a backend restart (hermes-agent 82603). - session.interrupt: on a failed host dial, probe the supervisor (new HostSupervisor.has_pending_turn) for a still-registered completion callback; when none is registered and the inflight turn is unchanged under the lock, clear running + inflight so Stop recovers the session. Fail closed when the probe itself errors or a pending completion exists: a live turn's own teardown must not be pre-empted. - the cancel/queue purge runs under history_lock even when the host dial raised before _interrupt_session_turn's own purge. - ported onto main's methods_session_interrupt.py split as _recover_unreachable_compute_host_turn; the in-process no-wait half is already covered by the previous commit. Co-authored-by: briandevans <252620095+briandevans@users.noreply.github.com>
182 lines
9.9 KiB
Python
182 lines
9.9 KiB
Python
"""Interrupt / steer / redirect session handlers (``methods_session`` split).
|
|
|
|
Moved verbatim from ``tui_gateway/methods_session.py`` (file-line ratchet #68779): the
|
|
bodies close over server.py globals through ``method_ctx.bind_module`` exactly as before —
|
|
``@method`` registration, ``_session_arg`` wrapping and module publication all still run
|
|
from the parent's ``register()`` via ``HandlerRegistry``.
|
|
"""
|
|
|
|
from .method_ctx import HandlerRegistry, bind_module
|
|
|
|
_registry = HandlerRegistry()
|
|
method = _registry.method
|
|
|
|
def _resume_wake_after_interrupt() -> None:
|
|
"""Re-arm a wake lease held by the interrupt caller or a voice capture.
|
|
|
|
``_wake_resume_if_owner`` no-ops unless that object holds the lease, so an
|
|
in-progress capture owned by someone else is not stolen. Interrupt already
|
|
silenced TTS before it can return an error; this matches that cut. A
|
|
``not_interrupted`` hosted-task mismatch must not call it.
|
|
"""
|
|
with _voice_sid_lock:
|
|
voice_owner = _voice_wake_owner
|
|
seen = []
|
|
for owner in (_caller_transport(), voice_owner):
|
|
if owner is None or any(owner is item for item in seen):
|
|
continue
|
|
seen.append(owner)
|
|
_wake_resume_if_owner(owner)
|
|
|
|
|
|
# ── interrupt / steer / redirect ─────────────────────────────────────
|
|
def _note_user_input(session: dict) -> None:
|
|
"""Mark the running turn as touched by the user (also covers compute-host turns, whose agent
|
|
flag never reaches this process); popped by the turn's own finally."""
|
|
with session["history_lock"]:
|
|
if session.get("running"):
|
|
session["_turn_user_input"] = True
|
|
|
|
|
|
def _recover_unreachable_compute_host_turn(sid: str, session: dict, inflight_at_interrupt) -> None:
|
|
"""Stop on a compute host that is gone (broken pipe, respawn exhausted, #82603).
|
|
|
|
The failed dial raised before ``_interrupt_session_turn`` purged the queue, and no ``turn.end`` will ever
|
|
clear ``running``, so every later submit would queue forever. Clear the turn only when the supervisor owes
|
|
no completion for it and the inflight dict is still the one this Stop saw (a successor turn keeps its own).
|
|
The probe runs outside ``history_lock``: compute-host calls never run under it.
|
|
"""
|
|
try:
|
|
pending = _get_compute_host_supervisor().has_pending_turn(sid)
|
|
except Exception as exc: # health: allow BLE001 -- an unreadable supervisor is not proof the turn ended; fail closed
|
|
logger.warning("session.interrupt: compute-host pending-turn probe failed for %s: %s", sid, exc)
|
|
pending = True
|
|
with session["history_lock"]:
|
|
session["_turn_cancel_requested"] = True
|
|
session["queued_prompt"] = None
|
|
session.pop("queued_prompts", None)
|
|
session["_queued_prompt_generation"] = int(session.get("_queued_prompt_generation", 0)) + 1
|
|
inflight_now = session.get("inflight_turn")
|
|
if not pending and session.get("running") and inflight_now is not None and inflight_now is inflight_at_interrupt:
|
|
session["running"] = False
|
|
_clear_inflight_turn(session)
|
|
_clear_pending(sid)
|
|
with contextlib.suppress(Exception):
|
|
_announce_cancelled_gateway_approvals(session, "interrupt", session_id=sid)
|
|
from tools.approval import resolve_gateway_approval
|
|
resolve_gateway_approval(session["session_key"], "deny", resolve_all=True)
|
|
|
|
|
|
@method("session.interrupt")
|
|
def _(rid, params: dict) -> dict:
|
|
_tts_stream_stop() # keypress barge-in also silences streaming TTS (voice is process-global)
|
|
resume_wake = True
|
|
try:
|
|
session, err = _sess_nowait(params, rid)
|
|
if err:
|
|
return err
|
|
if expected := _str_param(params, "expected_hosted_task_id"):
|
|
with session["history_lock"]:
|
|
task = session.get("_hosted_room_task")
|
|
if not (session.get("running") and isinstance(task, dict) and task.get("task_id") == expected):
|
|
resume_wake = False
|
|
return _ok(rid, {"status": "not_interrupted", "interrupted": False})
|
|
sid = str(params.get("session_id") or "")
|
|
_note_user_input(session)
|
|
if _session_uses_compute_host(session):
|
|
with session["history_lock"]:
|
|
inflight_at_interrupt = session.get("inflight_turn")
|
|
try:
|
|
_interrupt_session_turn(sid, session, request_id=f"interrupt-{rid}")
|
|
except Exception as exc: # health: allow BLE001 -- the bridge raises transport-specific errors unknown here; logged, and Stop recovers the session instead of failing
|
|
logger.warning("session.interrupt: compute-host interrupt failed for %s: %s", sid, exc)
|
|
_recover_unreachable_compute_host_turn(sid, session, inflight_at_interrupt)
|
|
return _ok(rid, {"status": "interrupted", "turn_isolation": True})
|
|
# No agent wait: a turn parked on the deferred build already honors _turn_cancel_requested. Waiting here
|
|
# failed Stop with 5032 once the build failed or outlived 30s (and started a build for a lazy session), so
|
|
# the Desktop, which interrupts before session.close, could not delete such a session (#133262).
|
|
_interrupt_session_turn(sid, session)
|
|
# Retire the crash-recovery marker NOW: until the run thread's finally, a backend exit looks like a crash
|
|
# and session.resume auto-continues the turn the user just stopped (the extra key covers compression
|
|
# rotating session_key mid-turn).
|
|
with session["history_lock"]:
|
|
active_marker_key = str(session.pop("_active_turn_marker_key", "") or "")
|
|
_retire_turn_marker(session, active_marker_key)
|
|
return _ok(rid, {"status": "interrupted"})
|
|
finally:
|
|
if resume_wake:
|
|
try:
|
|
_resume_wake_after_interrupt()
|
|
except Exception:
|
|
logger.debug("session.interrupt wake resume failed", exc_info=True)
|
|
|
|
|
|
def _apply_correction(rid, session: dict, verb: str, text: str, accepted_status: str) -> dict:
|
|
"""``agent.<verb>(text)``; on acceptance record it on the live turn (mid-turn resume rebuilds the bubble)
|
|
and purge queued self-copies so post-turn drain cannot re-fire the old prompt."""
|
|
try:
|
|
accepted = getattr(session["agent"], verb)(text)
|
|
except Exception as exc:
|
|
return _err(rid, 5000, f"{verb} failed: {exc}")
|
|
if accepted:
|
|
with session["history_lock"]:
|
|
_record_inflight_correction(session, text)
|
|
# #84417: steer does not cancel the live original, but a server queue self-copy of that original
|
|
# must still not re-fire after settle (same class as redirect).
|
|
# #84417: purge server-queue self-duplicates of the live original so post-turn drain cannot
|
|
# restart the pre-correction prompt.
|
|
_drop_queued_duplicates_of_inflight_user(session)
|
|
session["last_active"] = time.time()
|
|
return _ok(rid, {"status": accepted_status if accepted else "rejected", "text": text})
|
|
|
|
|
|
def _correction_method(name: str, verb: str, accepted_status: str, supported, unsupported: str):
|
|
"""steer/redirect RPC: ``params.text`` (4002, checked before the session) into a live session;
|
|
``supported(agent)`` gates 4010."""
|
|
@method(name)
|
|
def _(rid, params: dict) -> dict:
|
|
if not (text := (params.get("text") or "").strip()):
|
|
return _err(rid, 4002, "text is required")
|
|
session, err = _sess_nowait(params, rid)
|
|
if err:
|
|
return err
|
|
_note_user_input(session)
|
|
agent = session.get("agent")
|
|
# Redirect during the turn-build window (running=True, agent None): queue for the next turn instead of
|
|
# a misleading 4010 the client swallows into a lost follow-up.
|
|
if verb == "redirect" and agent is None and session.get("running"):
|
|
_enqueue_prompt(session, text, current_transport() or _stdio_transport)
|
|
session["last_active"] = time.time()
|
|
return _ok(rid, {"status": "queued", "text": text})
|
|
# Compression in flight: queue instead of steering/redirecting. A correction that
|
|
# reaches the provider mid-compression aborts the compression (explicit_interrupt)
|
|
# — the follow-up kills the turn that would answer it (#61042). Queued here, it
|
|
# drains when compression finishes (the Discord-gateway contract; mirrors the
|
|
# interrupt→queue demotion in gateway/run_busy.py for the channel busy path).
|
|
if _session_compression_in_flight(session):
|
|
_enqueue_prompt(session, text, current_transport() or _stdio_transport)
|
|
session["last_active"] = time.time()
|
|
return _ok(rid, {"status": "queued", "text": text})
|
|
if not supported(agent):
|
|
return _err(rid, 4010, unsupported)
|
|
# An idle agent accepts steer() but only the next turn drains it, spliced after an old tool
|
|
# row (#64578). 'rejected' makes the client queue it as a normal next prompt.
|
|
if verb == "steer" and not session.get("running"):
|
|
return _ok(rid, {"status": "rejected", "text": text})
|
|
return _apply_correction(rid, session, verb, text, accepted_status)
|
|
|
|
|
|
# Inject text into the next tool result without interrupting (AIAgent.steer(): no new user turn, no role
|
|
# alternation violation).
|
|
_correction_method("session.steer", "steer", "queued", lambda agent: hasattr(agent, "steer"),
|
|
"agent does not support steer")
|
|
# Redirect the active model turn while preserving valid work/context.
|
|
_correction_method("session.redirect", "redirect", "redirected",
|
|
lambda agent: getattr(agent, "_supports_active_turn_redirect", False) is True
|
|
and hasattr(agent, "redirect"), "agent does not support active-turn redirect")
|
|
|
|
|
|
def register(server) -> None:
|
|
"""Publish this module's helpers onto ``server`` (rebound to its globals) and install handlers."""
|
|
bind_module(globals(), server, skip=("_",))
|