mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
fix: /stop halts background subagents and returns their partial results as interrupted completions
A turn's hard interrupt fans out only to `_active_children`; background delegate_task units are detached from the parent at dispatch (`_dispatch_background` / honor_parent_interrupt=False), so /stop left them running to completion and their result arrived minutes later as a parked wake. Every stop surface now also calls `tools.async_delegation.interrupt_for_session` for the session's units: - gateway: `_interrupt_and_clear_session` (busy /stop, /new share it — /new already did this in `_handle_reset_command`, the second request is idempotent) and the idle `_handle_stop_command` tail, which replied "No active task to stop." while a background child was running; it now stops them and replies "Stopped". - tui_gateway `session.interrupt` (Desktop Stop / TUI stop) — own UI sid + spawner id only, so a viewer tab never kills gateway work. - acp_adapter `cancel`. - CLI `/stop` already used `interrupt_all` (process-wide); unchanged. The stop recurses: depth>0 delegations are always synchronous (`_model_background_value`), so the child's hard interrupt reaches its workers through its own `_active_children` fan-out, and each level's interrupted partial result rolls up as that child's completion. An interrupted child's entry now carries what it actually had: the loop's `final_response` is the "Operation interrupted." placeholder (also appended as the closing assistant row), so `_build_result_entry` takes the child's last real assistant text as `summary` and keeps the placeholder as `error`. The unit finalizes normally and re-enters at once as its completion notice (status=interrupted, "Partial output: ...", "Subagent Task Interrupted" on TUI/Desktop) instead of the chat waiting for the child's budget to run out. Docs: delegate_task description, tools/AGENTS.md, delegation.md, gateway-session-lifecycle.md. Part of #114456
This commit is contained in:
1 parent
2b94b0d40f
commit
b71359066c
11 files changed
+170
-11
No files matched your search
@@ -638,6 +638,10 @@ class HermesACPAgent(SlashCommandsMixin, acp.Agent):
|
||||
try:
|
||||
if state.agent:
|
||||
request_hard_interrupt(state.agent)
|
||||
# Background delegations are detached from the turn's fan-out; they end with the cancel.
|
||||
from tools.async_delegation import interrupt_for_session
|
||||
interrupt_for_session(parent_session_id=str(getattr(state.agent, "session_id", "") or ""),
|
||||
reason="acp_cancel")
|
||||
except Exception:
|
||||
logger.debug("Failed to interrupt ACP session %s", session_id, exc_info=True)
|
||||
logger.info("Cancelled session %s", session_id)
|
||||
|
||||
@@ -481,6 +481,13 @@ class GatewayAgentCacheMixin:
|
||||
session_key, interrupt_reason=interrupt_reason, invalidation_reason=invalidation_reason,
|
||||
)
|
||||
from gateway.run import _AGENT_PENDING_SENTINEL
|
||||
# The turn's hard interrupt reaches only its in-turn children; background delegations were
|
||||
# detached at dispatch and would otherwise run to completion and wake the session later.
|
||||
# Each interrupted unit still returns as a completion (status=interrupted, partial output).
|
||||
from tools.async_delegation import interrupt_for_session
|
||||
interrupt_for_session(
|
||||
session_key=session_key, reason=invalidation_reason,
|
||||
parent_session_id=str(getattr(running_agent, "session_id", "") or ""))
|
||||
if running_agent and running_agent is not _AGENT_PENDING_SENTINEL:
|
||||
# Plugins holding a per-turn external resource (an outbound RPC blocked on a tool result
|
||||
# the loop will never consume) learn the turn is gone. Fires for /stop and the /new
|
||||
|
||||
@@ -458,7 +458,13 @@ class GatewaySlashCommandsMixin(
|
||||
reason, session_key, len(fallback_keys), ", ".join(fallback_keys))
|
||||
return EphemeralReply(t("gateway.stop.stopped"))
|
||||
|
||||
# No running agent anywhere for this scope. A platform status indicator can still be stuck —
|
||||
# No running agent anywhere for this scope. Background delegations the session dispatched in an
|
||||
# earlier turn still count as "active": stop them; each returns as an interrupted completion.
|
||||
from tools.async_delegation import interrupt_for_session
|
||||
if interrupt_for_session(session_key=session_key, reason="stop_command",
|
||||
parent_session_id=str(getattr(session_entry, "session_id", "") or "")):
|
||||
return EphemeralReply(t("gateway.stop.stopped"))
|
||||
# A platform status indicator can still be stuck —
|
||||
# e.g. Slack's persistent assistant.threads.setStatus survives a gateway restart or a turn
|
||||
# that died without a final send.
|
||||
# Best-effort clear so /stop always dismisses a phantom "is thinking...". See #32295.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Invariant: ``/stop`` ends the session's BACKGROUND delegate_task children too — whether the session is
|
||||
mid-turn (busy fast path) or idle after the dispatching turn already ended — instead of only the in-turn
|
||||
children, leaving the detached unit to run to completion and wake the chat later. See #114456.
|
||||
|
||||
Real ``GatewayRunner`` + real ``BasePlatformAdapter`` subclass; the background unit is a real
|
||||
``tools.async_delegation`` registry record whose ``interrupt_fn`` is the observable.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.async_delegation as ad
|
||||
from gateway.config import GatewayConfig, Platform, PlatformConfig
|
||||
from gateway.platforms.base import BasePlatformAdapter, SendResult
|
||||
from gateway.platforms.event import MessageEvent, MessageType
|
||||
from gateway.run import GatewayRunner
|
||||
from gateway.session import SessionSource
|
||||
|
||||
|
||||
class _Adapter(BasePlatformAdapter):
|
||||
def __init__(self):
|
||||
super().__init__(PlatformConfig(enabled=True, token="x"), Platform.TELEGRAM)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "telegram"
|
||||
|
||||
async def connect(self, *, is_reconnect=False):
|
||||
return True
|
||||
|
||||
async def disconnect(self):
|
||||
pass
|
||||
|
||||
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
||||
return SendResult(success=True)
|
||||
|
||||
async def get_chat_info(self, chat_id):
|
||||
return {"id": chat_id, "type": "private"}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_async_delegation():
|
||||
ad._reset_for_tests()
|
||||
yield
|
||||
ad._reset_for_tests()
|
||||
|
||||
|
||||
def _seed_unit(session_key: str, parent_session_id: str = "") -> MagicMock:
|
||||
"""A live background unit as ``_dispatch_background`` registers it (running, routed by session_key)."""
|
||||
fn = MagicMock()
|
||||
with ad._records_lock:
|
||||
ad._records["deleg_bg1"] = {"delegation_id": "deleg_bg1", "status": "running", "session_key": session_key,
|
||||
"origin_ui_session_id": "", "parent_session_id": parent_session_id, "interrupt_fn": fn}
|
||||
return fn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("session_state", ["idle", "busy"])
|
||||
async def test_stop_ends_background_delegations_of_the_session(monkeypatch, session_state):
|
||||
monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "u1")
|
||||
adapter = _Adapter()
|
||||
runner = GatewayRunner(config=GatewayConfig(platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="x")}))
|
||||
runner.adapters = {Platform.TELEGRAM: adapter}
|
||||
source = SessionSource(platform=Platform.TELEGRAM, chat_id="c1", chat_type="dm", user_id="u1", user_name="tester")
|
||||
key = adapter._event_session_key(MessageEvent(text="", message_type=MessageType.TEXT, source=source))
|
||||
stop_fn = _seed_unit(key)
|
||||
other_fn = MagicMock()
|
||||
with ad._records_lock:
|
||||
ad._records["deleg_other"] = {"delegation_id": "deleg_other", "status": "running", "session_key": "agent:main:telegram:dm:c2",
|
||||
"origin_ui_session_id": "", "parent_session_id": "", "interrupt_fn": other_fn}
|
||||
|
||||
if session_state == "busy":
|
||||
await runner._interrupt_and_clear_session(key, source, interrupt_reason="stop", invalidation_reason="stop_command")
|
||||
else:
|
||||
reply = await runner._handle_stop_command(MessageEvent(text="/stop", message_type=MessageType.TEXT, source=source))
|
||||
# The chat is told something WAS stopped, not "No active task to stop."
|
||||
assert "Stopped" in str(getattr(reply, "text", reply))
|
||||
|
||||
stop_fn.assert_called_once()
|
||||
other_fn.assert_not_called() # another chat's background work is untouched
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Invariant: a child stopped mid-work reports what it HAD so far. The loop's ``final_response`` for an
|
||||
interrupted turn is the placeholder ``"Operation interrupted."`` (also appended as the closing assistant row);
|
||||
the parent-visible entry must carry the child's last real assistant text as ``summary`` and keep the
|
||||
placeholder as ``error`` — never a partial result that reads as an empty stop. See #114456.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tools.delegate_tool_child_run import _SchemaOutcome, _build_result_entry
|
||||
|
||||
|
||||
def test_interrupted_child_entry_carries_its_partial_output():
|
||||
messages = [
|
||||
{"role": "user", "content": "kickoff"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Audited 3 of 7 modules; two findings so far."}],
|
||||
"tool_calls": [{"id": "c1", "type": "function", "function": {"name": "terminal", "arguments": "{}"}}]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": "[Command interrupted]"},
|
||||
{"role": "assistant", "content": "Operation interrupted."},
|
||||
]
|
||||
result = {"final_response": "Operation interrupted.", "messages": messages, "api_calls": 2,
|
||||
"completed": False, "interrupted": True}
|
||||
child = SimpleNamespace(model="m", session_estimated_cost_usd=0.0, session_cost_status="unknown",
|
||||
session_prompt_tokens=1, session_completion_tokens=1, _delegate_role="leaf")
|
||||
|
||||
entry = _build_result_entry(child, result, 0, 12.5, _SchemaOutcome(None, None, [], 0))
|
||||
|
||||
assert entry["status"] == entry["exit_reason"] == "interrupted"
|
||||
assert entry["summary"] == "Audited 3 of 7 modules; two findings so far."
|
||||
assert entry["error"] == "Operation interrupted."
|
||||
+6
-1
@@ -140,7 +140,12 @@ processes are killed at its teardown and their notices are suppressed in the par
|
||||
(`process_registry.transfer_ownership`) so the completion routes and reaps by the new owner; un-handed leftovers land on
|
||||
the result as `orphaned_processes`, exited-but-never-read notify processes as `unread_completions` (`_ChildRun.account_background_processes`, before `cleanup` kills them). **Child kernels:** a child's `execute_code` kernels are keyed `<parent-owner>::child::<child-session-id>`, pinned against the `max_session_kernels` LRU cap while the child runs and disposed by `cleanup` (`code_kernel.shutdown_kernels_for_delegated_child`) — never let a finished child's kernel squat the cap. **output_schema:** a miss after the one retry keeps `status: completed` with the raw text in `summary` plus `schema_valid: false` / `schema_errors` / `schema_note` — never discard a child's result. **Durability:** background
|
||||
delegation is process-local; work that must survive restart uses `cronjob` or
|
||||
`terminal(background=True, notify_on_complete=True)`. API: `website/docs/developer-guide/subagent-lifecycle-api.md`.
|
||||
`terminal(background=True, notify_on_complete=True)`. **Stop fan-out:** a turn's hard interrupt reaches only
|
||||
`_active_children`; background units are detached at dispatch, so every stop surface (gateway `/stop` busy AND
|
||||
idle paths, TUI `session.interrupt`, ACP `cancel`, CLI `/stop` via `interrupt_all`) calls
|
||||
`async_delegation.interrupt_for_session` too. Depth>0 delegations are always synchronous
|
||||
(`_model_background_value`), so the stop recurses through the child's own `_active_children` fan-out; an
|
||||
interrupted entry's `summary` is the child's last real assistant text (`_build_result_entry`). API: `website/docs/developer-guide/subagent-lifecycle-api.md`.
|
||||
|
||||
## Tests
|
||||
|
||||
|
||||
@@ -545,16 +545,14 @@ _DESCRIPTION_HEAD = (
|
||||
"as a new message when subagents finish ({delivery}). Background results are delivered only "
|
||||
"BETWEEN your turns: finish whatever does not depend on them, then give a one-line status and END YOUR TURN. Never "
|
||||
"wait or poll on transcripts, artifact files, or CI for a child. "
|
||||
"While children run, `action` (list/steer/stop) controls them live — steer when a transcript shows a "
|
||||
"child drifting.\n\n"
|
||||
"USE FOR: reasoning-heavy subtasks, work that would flood your context with intermediate data, or independent "
|
||||
"parallel workstreams.\n"
|
||||
"While children run, `action` (list/steer/stop) controls them live.\n\n"
|
||||
"USE FOR: reasoning-heavy subtasks, work that would flood your context, or independent parallel workstreams.\n"
|
||||
"DO NOT USE FOR (use these instead):\n"
|
||||
"- Mechanical multi-step work with no reasoning needed -> execute_code\n"
|
||||
"- A single tool call -> call the tool directly\n"
|
||||
"- Tasks needing user interaction -> subagents cannot ask questions\n"
|
||||
"- Durable work that must survive this session -> cronjob or terminal(background=True, notify=True); /stop, /new, "
|
||||
"or process exit discards running subagents.\n\n"
|
||||
"or process exit halts running subagents (whole tree); each returns an 'interrupted' completion with partial output.\n\n"
|
||||
"RULES:\n"
|
||||
"- Children know nothing of this conversation: pass everything needed via 'context', including any required "
|
||||
"output language, tone, or style (e.g. \"respond in Chinese\").\n"
|
||||
|
||||
@@ -503,8 +503,18 @@ def _build_result_entry(
|
||||
# "(empty)" is run_agent's give-up sentinel after repeated empty LLM
|
||||
# responses (usually a transport bug) — a failure, not a success.
|
||||
usable_summary = bool(summary) and summary.strip() != "(empty)"
|
||||
interrupt_note = ""
|
||||
if result.get("interrupted", False):
|
||||
status, exit_reason = "interrupted", "interrupted"
|
||||
# The loop's final_response is a placeholder here ("Operation interrupted…", also appended as the closing
|
||||
# assistant row); the completion must carry what the child actually had so far — its last real assistant
|
||||
# text — and keep the placeholder as the error.
|
||||
from agent.message_content import flatten_message_text
|
||||
placeholders = {"", summary.strip(), "Operation interrupted."}
|
||||
partial = next((t for m in reversed(result.get("messages") or []) if m.get("role") == "assistant"
|
||||
and (t := flatten_message_text(m.get("content")).strip()) not in placeholders), "")
|
||||
if partial:
|
||||
interrupt_note, summary = summary.strip(), partial
|
||||
elif result.get("failed") or result.get("error"):
|
||||
# The loop returns the error text as final_response, which would otherwise read as "completed". Never report a
|
||||
# provider rejection as "max_iterations" — that is only truthful for real budget exhaustion.
|
||||
@@ -554,6 +564,8 @@ def _build_result_entry(
|
||||
_failure_reason = result.get("failure_reason")
|
||||
if isinstance(_failure_reason, str) and _failure_reason:
|
||||
entry["failure_reason"] = _failure_reason
|
||||
elif interrupt_note:
|
||||
entry["error"] = interrupt_note
|
||||
|
||||
# Schema-validation outcome — emitted ONLY when a schema was requested, so
|
||||
# legacy (schema-less) payloads keep their exact shape.
|
||||
|
||||
@@ -621,6 +621,14 @@ def _interrupt_session_turn(sid: str, session: dict, *, request_id: str | None =
|
||||
if should_interrupt:
|
||||
from agent.interrupt_compat import request_hard_interrupt
|
||||
request_hard_interrupt(session.get("agent"))
|
||||
# Background delegations are detached from the turn's interrupt fan-out; a stop ends them too
|
||||
# (own UI sid + spawner id only — a viewer tab must not kill gateway work). Each returns as an
|
||||
# interrupted completion with its partial output.
|
||||
with contextlib.suppress(Exception):
|
||||
from tools.async_delegation import interrupt_for_session
|
||||
interrupt_for_session(
|
||||
origin_ui_session_id=_lifecycle_own_sid(session, sid), reason="user_stop",
|
||||
parent_session_id=str(getattr(session.get("agent"), "session_id", "") or ""))
|
||||
if not run_thread_alive:
|
||||
with session["history_lock"]:
|
||||
if session.get("running"):
|
||||
|
||||
@@ -463,6 +463,17 @@ post-command drain starts it right away instead of the session idling until the
|
||||
message. Whether a wake pinned to a session that `/new` just closed may still run is decided at
|
||||
processing time (`_resolve_async_delegation_session`, fail-closed).
|
||||
|
||||
Both commands also end the session's **background delegations** (`tools.async_delegation.
|
||||
interrupt_for_session`, selected by routing key and by the spawner's durable session id):
|
||||
`_interrupt_and_clear_session` fans the stop out for the busy path, and `_handle_stop_command`
|
||||
does the same for an idle session whose dispatching turn already ended (replying "Stopped" rather
|
||||
than "No active task to stop"). The turn's own hard interrupt never reaches those units — they are
|
||||
detached from `_active_children` at dispatch — so without the fan-out they run to completion and wake
|
||||
the chat minutes later. Each stopped unit still finalizes normally and re-enters as its completion
|
||||
notice with `status="interrupted"` and the child's partial output. `/new` and `/reset` already did this
|
||||
in `_handle_reset_command`; the shared helper's earlier call is idempotent there (a hard interrupt
|
||||
requested twice is one stop).
|
||||
|
||||
### FIFO Invariant
|
||||
|
||||
Each `/queue` invocation produces exactly one full agent turn, in FIFO order, with no
|
||||
|
||||
@@ -212,7 +212,7 @@ The dispatch handle lists each unit (`units[].delegation_id`, `group`, `task_ind
|
||||
- **Thread pool:** Uses `ThreadPoolExecutor` with the configured concurrency limit as max workers
|
||||
- **Progress display:** In CLI mode, a tree-view shows tool calls from each subagent in real-time with per-task completion lines. In gateway mode, progress is batched and relayed to the parent's progress callback. CLI and TUI completion notices use task-first titles such as `Subagent Task Completed: Review changes`; multi-task groups use the group name and task count. Unsuccessful or incomplete work gets a corresponding status label. These compact notices do not replace the full results delivered to the parent agent.
|
||||
- **Result ordering:** Within a unit, results are sorted by task index to match input order regardless of completion order; `TASK i/N` labels index the whole call
|
||||
- **Cancellation:** Follow-up messages do not cancel a top-level background batch. `/stop` or closing/resetting the owning session cancels its active children. Synchronous orchestrator children still follow their parent's interrupt state
|
||||
- **Cancellation:** Follow-up messages do not cancel a top-level background batch. `/stop` (gateway `/stop`, CLI `/stop`, the Desktop/TUI Stop button, an ACP cancel) or closing/resetting the owning session ends its background children and every synchronous descendant beneath them; each stopped child still returns as a completion with `status="interrupted"` and its partial output
|
||||
|
||||
Synchronous single-task delegation from an orchestrator runs directly without thread pool overhead.
|
||||
|
||||
@@ -548,11 +548,11 @@ delegate_task(
|
||||
:::warning Background completion durability is not durable execution
|
||||
Top-level model-facing `delegate_task` calls run in the background automatically where the session supports later delivery. Hermes returns a handle immediately, and the result re-enters the conversation after the child or batch finishes. Orchestrator subagents wait for their workers in the current turn because they must synthesize those results before returning. Stateless request/response endpoints fall back to synchronous execution when they cannot deliver a detached result later.
|
||||
|
||||
- Normal follow-up messages do not cancel background children. `/stop` cancels running background delegations, and closing or resetting the owning session discards its active children.
|
||||
- Normal follow-up messages do not cancel background children. `/stop` on any surface (gateway `/stop` — also when the session is idle after the dispatching turn ended — CLI `/stop`, the Desktop/TUI Stop button, an ACP cancel) ends the session's running background delegations, and closing or resetting the owning session does the same.
|
||||
- Explicit session close/reset interrupts that session's background children. Closing a TUI viewer of a gateway-owned session does not kill the gateway's work.
|
||||
- A Hermes process restart does **not** resume a running child. Its attempt becomes `unknown` because Hermes cannot prove which side effects happened.
|
||||
- A child that completed before restart but whose result was not delivered is restored and routed back through the owning session's normal checks.
|
||||
- Cancelled children return a structured result (`status="interrupted"`, `exit_reason="interrupted"`), but because the parent was interrupted too, that result often never makes it into a user-visible reply.
|
||||
- Stopped children return a structured result (`status="interrupted"`, `exit_reason="interrupted"`) whose `summary` is the last text the child produced before the stop (the interrupt placeholder moves to `error`). The stop recurses down the spawn tree — an orchestrator child's synchronous workers are interrupted first and their partial results roll up into the child's own interrupted result — and each background unit's result re-enters the conversation right away as its normal completion notice (`Subagent Task Interrupted: …`), so nothing waits for the child to exhaust its budget.
|
||||
|
||||
For **durable execution** that must survive session closure or process restart, use:
|
||||
|
||||
@@ -566,7 +566,7 @@ For **durable execution** that must survive session closure or process restart,
|
||||
- Subagents inherit the parent's enabled toolsets; the model cannot select or widen them per call
|
||||
- **Nested delegation is opt-in** — only `role="orchestrator"` children can delegate further, and only when `max_spawn_depth` is raised from its default of 1 (flat). Disable globally with `orchestrator_enabled: false`.
|
||||
- Leaf subagents **cannot** call: `delegate_task`, `clarify`, `memory`, `send_message`, `cronjob`. Orchestrator subagents retain `delegate_task` but keep the other blocks. Both roles retain `execute_code` (programmatic tool calling) so children can batch mechanical work instead of burning reasoning iterations.
|
||||
- **Cancellation follows ownership** — `/stop` or closing/resetting the owning session cancels its background children; synchronous descendants under orchestrators follow their parent's interrupt state
|
||||
- **Cancellation follows ownership** — `/stop` or closing/resetting the owning session ends its background children and the synchronous descendants beneath them; each returns an interrupted completion carrying its partial output
|
||||
- Only the final summary enters the parent's context, keeping token usage efficient
|
||||
- Subagents inherit the parent's **API key, provider configuration, and credential pool** (enabling key rotation on rate limits)
|
||||
|
||||
|
||||
Reference in new issue
Block a user