hermes-agent/hermes_cli/plugins_dispatch.py

"""Plugin hook / middleware / event-bus / system-prompt-section dispatch.

Mixed into :class:`hermes_cli.plugins.PluginManager`. ``_resolve_hook_callback_timeout`` stays on
the origin (tests patch it there) and is looked up lazily.
"""

from __future__ import annotations

import asyncio
import contextvars
import copy
import inspect
import logging
import queue
import re
import threading
import time
import types
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union

from hermes_cli.middleware import OBSERVER_SCHEMA_VERSION

logger = logging.getLogger("hermes_cli.plugins")

# Allowlist of agent-turn hot-path hooks bounded by plugins.hook_callback_timeout (fail-open:
# abandon without join — joining reintroduced a shutdown hang). Unlisted hooks run synchronously.
# Intentionally unbounded: on_session_finalize/reset (last-chance flush — abandon can lose state);
# subagent_start (observer); pre_gateway_dispatch (policy gate — neither fail mode is acceptable);
# pre/post_approval_* (approval UX has its own timeout); kanban_* (own heartbeat/stale reclaim).
# The goal is to stop a hung Python plugin callback from wedging the conversation loop (#76821) without
# joining the worker (avoids the #6622 ThreadPoolExecutor shutdown hang). Hooks not listed below run
# synchronously to completion. (on_session_start/end stay bounded — they sit on the common session-boundary
# path.) - subagent_start — observer only; blocking delegation belongs in pre_tool_call. Lower frequency
# than tool/LLM hooks. Abandoning is unsafe either way (fail-open skips auth-like checks; fail-closed can
# drop legitimate messages). Prefer finish-or-exception fallthrough. - pre_approval_request /
# post_approval_response — observers only (cannot veto); the approval UX already has its own timeout; not on
# the tool loop hot path. - kanban_task_* — fire after the board DB commit, observers only, in
# dispatcher/worker processes; kanban has its own heartbeat/stale reclaim. Abandon-without-join also leaves
# a daemon thread that may still mutate shared state — safer for value-returning observers than for
# gates/flushes.
_HOOK_TIMEOUT_BOUNDED_HOOKS: set[str] = {
    "post_tool_call", "transform_terminal_output", "transform_tool_result", "transform_llm_output",
    "pre_llm_call", "post_llm_call", "pre_api_request", "post_api_request", "api_request_error",
    "pre_auxiliary_call", "post_auxiliary_call", "pre_verify", "on_session_start", "on_session_end",
    # Fail-open consumer on every inbound gateway message: a hung plugin must not stall the profile.
    "post_gateway_admission",
    # Observer on every local session's commands, fired by the gateway (gateway/session_commands.py).
    "pre_command",
    # Fires inside the compression commit window (lease held): a hung plugin must not wedge the session.
    "post_compaction",
}
# Hooks whose calls are told apart by another payload field: one session's commands are one
# sequence, so a still-running callback skips that session's next command, not another session's.
_HOOK_CALL_IDENTITY_FIELDS: dict[str, tuple[str, ...]] = {
    "pre_command": ("session_key",), "post_compaction": ("session_id",),
}

# Policy hooks: timeout / still-running must fail closed (block the tool).
_HOOK_TIMEOUT_FAIL_CLOSED_HOOKS: set[str] = {"pre_tool_call"}
# Documented parent-thread serialization contract — never run on a timeout worker (hooks.md).
_HOOK_CALLER_THREAD_HOOKS: set[str] = {"subagent_stop"}
# After a timeout, suppress the same callback this long so a hung hook cannot pile up threads.
_HOOK_TIMEOUT_SUPPRESSION_SECONDS = 60.0
# Live workers a hung callback may accumulate before it is skipped outright (#105223 / #98382).
_HOOK_MAX_ABANDONED_WORKERS = 3
_PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE = "pre_tool_call plugin callback timed out or is still running"

# The one hook whose callback rewrites the reply the user would otherwise see streaming.
STREAM_HOLDING_HOOK = "transform_llm_output"


def stream_held() -> bool:
    """True when a plugin loaded in the active profile registered ``transform_llm_output`` with
    ``hold_stream=True``: that profile's replies reach the user only as the rewritten final.
    Lazy-discovers first, like ``has_hook``."""
    from hermes_cli.plugins import _delivery_manager

    return _delivery_manager().holds_stream()


def held_stream_transforms() -> tuple[Callable, ...]:
    """:meth:`PluginDispatchMixin.held_transforms` of the active profile's manager (lazy-discovers)."""
    from hermes_cli.plugins import _delivery_manager

    return _delivery_manager().held_transforms()


def _policy_error_block_directive(hook_name: str, cb: Callable, exc: BaseException) -> dict[str, str]:
    """Block directive for a fail-closed hook whose callback raised: names the callback and the
    error (truncated — a hook that embeds tool args in its exception must not grow the tool
    result) so the operator can tell a crashing guard from a slow one."""
    callback_name = getattr(cb, "__name__", repr(cb))
    return {"action": "block",
            "message": f"{hook_name} plugin callback {callback_name} raised {type(exc).__name__}: {str(exc)[:200]}"}

# System-prompt sections are tightly bounded: they become high-trust prompt bytes charged every turn.
SYSTEM_PROMPT_SECTION_POSITIONS = frozenset({"after_memory"})
DEFAULT_SYSTEM_PROMPT_SECTION_MAX_CHARS = 4_000
MAX_SYSTEM_PROMPT_SECTION_CHARS = 4_000
MAX_SYSTEM_PROMPT_SECTIONS = 32
MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS = 8_000
_SYSTEM_PROMPT_SECTION_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
_SYSTEM_PROMPT_SECTION_HEADING_PREFIX = "## Plugin Context: "
PLUGIN_SECTIONS_START = "<!-- hermes-plugin-sections:start -->"
PLUGIN_SECTIONS_END = "<!-- hermes-plugin-sections:end -->"


def is_valid_system_prompt_section_id(value: Any) -> bool:
    """Return whether *value* is a stable, heading-safe section identifier."""
    return isinstance(value, str) and bool(_SYSTEM_PROMPT_SECTION_ID_RE.fullmatch(value))


def format_system_prompt_section(section_id: str, content: str) -> str:
    """Render an auditable, length-framed block recoverable from the full prompt."""
    return (
        f"{_SYSTEM_PROMPT_SECTION_HEADING_PREFIX}{section_id}\n"
        f"<!-- hermes-plugin-section-chars:{len(content)} -->\n\n{content}")


def format_system_prompt_sections(sections: list) -> str:
    """Render the canonical container used for persistence recovery."""
    if not sections:
        return ""
    blocks = [format_system_prompt_section(item.id, item.content) for item in sections]
    return f"{PLUGIN_SECTIONS_START}\n" + "\n\n".join(blocks) + f"\n{PLUGIN_SECTIONS_END}"


# Reserved event namespace prefix — only core may publish ``hermes:<event>``.
HERMES_EVENT_NAMESPACE = "hermes"
# Event recursion depth cap (subscribers may emit); over-deep emits are dropped with a warning.
_EVENT_EMIT_DEPTH_CAP = 8
# Max queued + running events per manager generation; emit never waits — a full budget drops.
_EVENT_PENDING_CAP = 64
_EVENT_WORKER_STOP = object()


@dataclass(frozen=True)
class PluginSystemPromptSection:
    """A plugin-owned section rendered once for each new session."""

    id: str
    content: str | Callable[[Mapping[str, Any]], str]
    position: str
    max_chars: int
    plugin: str


@dataclass(frozen=True)
class RenderedPluginSystemPromptSection:
    """Validated prompt bytes frozen on the owning AIAgent."""

    id: str
    content: str
    position: str
    plugin: str


@dataclass(frozen=True)
class _EventSubscription:
    """Host-owned subscription ledger entry."""

    owner: str
    callback: Callable


@dataclass(frozen=True)
class _QueuedPluginEvent:
    """Immutable dispatch envelope consumed by the event worker."""

    event: str
    payload: dict[str, Any]
    subscriptions: tuple[_EventSubscription, ...]
    depth: int
    generation: int
    # The emitter's contextvars: the single worker thread serves every profile, so each delivery
    # runs under the profile scope the emit happened in (#118538).
    context: contextvars.Context


# Hook callback timeout (non-blocking abandon). Default cap per Python hook callback; overridden by
# ``plugins.hook_callback_timeout``. Shell hooks enforce their own subprocess timeout.
_HOOK_CALLBACK_TIMEOUT_SECS = 30.0
_MAX_HOOK_CALLBACK_TIMEOUT_SECS = 600.0
_HOOK_SKIPPED = object()  # returned by _run_hook_callback_bounded on skip/timeout


def _hook_call_identity(kwargs: dict[str, Any], hook_name: str = "") -> Optional[str]:
    """Identity of the call this callback fires for, or ``None`` when the event has none.

    Concurrent invocations of the same tool in one session must not collapse into one
    gate key: they are different work, and treating the second as a duplicate drops the
    hook as if a callback had timed out (upstream #98382). The identity is already in the
    payload; nothing new is plumbed. Deliberately not ``api_request_id`` — one API request
    carries many tool calls, which would re-collapse the keys.
    """
    for field in _HOOK_CALL_IDENTITY_FIELDS.get(hook_name, ("tool_call_id", "turn_id")):
        value = kwargs.get(field)
        if isinstance(value, str) and value:
            return value
    return None


def _hook_uses_callback_timeout(hook_name: str, timeout: float) -> bool:
    """Whether *hook_name* should run under the non-blocking timeout path."""
    if timeout <= 0 or hook_name in _HOOK_CALLER_THREAD_HOOKS:
        return False
    return hook_name in _HOOK_TIMEOUT_BOUNDED_HOOKS or hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS


_PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS = 30.0


def resolve_plugin_command_result(result: Any) -> Any:
    """Resolve a plugin command result, awaiting async handlers: ``asyncio.run`` when no loop is
    running, else a helper thread with its own loop (30s bound so a hung handler cannot wedge the
    terminal)."""
    if not inspect.isawaitable(result):
        return result
    try:
        asyncio.get_running_loop()
    except RuntimeError:
        return asyncio.run(result)
    outcome: dict[str, Any] = {}
    failure: dict[str, BaseException] = {}
    done = threading.Event()

    def _runner() -> None:
        try:
            outcome["value"] = asyncio.run(result)
        except BaseException as exc:  # pragma: no cover - re-raised below
            failure["exc"] = exc
        finally:
            done.set()

    # copy_context: the helper thread must see the caller's profile/secret scope, else an
    # async hook under a running loop reads the default HERMES_HOME and get_secret raises.
    threading.Thread(target=contextvars.copy_context().run, args=(_runner,),
                     name="hermes-plugin-command-await", daemon=True).start()
    if not done.wait(timeout=_PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS):
        raise TimeoutError("Plugin command async handler did not complete within "
                           f"{_PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS:.0f}s")
    if "exc" in failure:
        raise failure["exc"]
    return outcome.get("value")


class PluginDispatchMixin:
    @staticmethod
    def _hook_callback_kwargs(callback: Callable, payload: dict[str, Any]) -> dict[str, Any]:
        """The slice of *payload* a callback accepts: everything for ``**kwargs`` (or
        un-introspectable) callbacks, only declared names for narrow legacy signatures."""
        try:
            parameters = inspect.signature(callback).parameters
        except (TypeError, ValueError):
            return dict(payload)  # no introspectable signature
        if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
            return dict(payload)
        keyword_kinds = {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY}
        return {
            name: value for name, value in payload.items()
            if name in parameters and parameters[name].kind in keyword_kinds
        }

    @classmethod
    def _invoke_hook_callback(cls, callback: Callable, payload: dict[str, Any]) -> Any:
        """Invoke a hook while withholding additive fields from narrow legacy callbacks.

        An ``async def`` callback returns a coroutine; resolve it the way plugin slash commands
        are (loop-safe), otherwise the bare coroutine object is appended to the results and the
        plugin's body never runs (#12449).
        """
        return resolve_plugin_command_result(callback(**cls._hook_callback_kwargs(callback, payload)))

    def invoke_hook(self, hook_name: str, **kwargs: Any) -> list[Any]:
        """Call all callbacks for *hook_name*; return their non-``None`` results.

        Payloads evolve additively: ``**kwargs`` callbacks get everything, narrow signatures only
        what they declare. Each callback is isolated. Bounded hooks and ``pre_tool_call`` run under
        ``plugins.hook_callback_timeout`` (worker abandoned, never joined); ``pre_tool_call`` fails
        closed with a block directive, others skip. ``_HOOK_CALLER_THREAD_HOOKS`` always run on the
        caller thread. ``pre_llm_call`` may return ``{"context": "..."}`` (or a str) to inject.
        """
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return []
        # Gateway platform events define event-local envelopes; a bus-wide version here would turn
        # unrelated adapter payloads into one monolithic compatibility contract.
        if hook_name != "gateway_platform_event":
            kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
        return self._run_hook_callbacks(hook_name, self._hooks.get(hook_name, []), kwargs)

    def _run_hook_callbacks(self, hook_name: str, callbacks: Any, kwargs: dict[str, Any],
                            failed: Optional[list] = None) -> list[Any]:
        """:meth:`invoke_hook`'s loop over ``callbacks``; each one that raised or timed out is
        appended to ``failed`` when given."""
        from hermes_cli.plugins import _resolve_hook_callback_timeout
        results: list[Any] = []
        timeout = _resolve_hook_callback_timeout()
        use_timeout = _hook_uses_callback_timeout(hook_name, timeout)
        fail_closed = hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS
        for cb in callbacks:
            try:
                if use_timeout:
                    ret = self._run_hook_callback_bounded(hook_name, cb, kwargs, timeout)
                    if ret is _HOOK_SKIPPED:
                        if failed is not None:
                            failed.append(cb)
                        if fail_closed:  # policy hook: fail closed with a block directive
                            results.append({"action": "block", "message": _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE})
                        continue
                else:
                    ret = self._invoke_hook_callback(cb, kwargs)
                if ret is not None:
                    results.append(ret)
            except (Exception, SystemExit) as exc:  # health: allow BLE001 -- moved from invoke_hook: plugin callbacks are untrusted; each failure is reported and isolated
                self._report_hook_failure(hook_name, cb, kwargs, exc)
                if failed is not None:
                    failed.append(cb)
                if fail_closed:  # a guard that raised made no decision: same veto as a timeout
                    results.append(_policy_error_block_directive(hook_name, cb, exc))
        return results

    def _report_hook_failure(
        self, hook_name: str, cb: Callable, kwargs: dict[str, Any], exc: BaseException, *, surface: str = "Hook"
    ) -> None:
        """One WARNING per distinct (hook, callback, error); identical repeats at DEBUG.

        A callback whose signature names a parameter the hook never sends (``tool_data`` instead
        of ``tool_name``/``args``) fails identically on every tool call — ~1700 WARNING lines an
        hour that bury real signals (#111922). The first report names the fields the hook does
        provide so the plugin author can fix the signature. The key names the callback by
        module/qualname (not ``id()``, which CPython recycles across plugin reloads) and
        truncates the message so a hook that embeds tool args in its error cannot grow the set
        per call; the set is cleared on unload alongside the timeout-suppression map.
        """
        callback_name = getattr(cb, "__name__", repr(cb))
        key = (hook_name, getattr(cb, "__module__", ""), getattr(cb, "__qualname__", callback_name),
               type(exc).__name__, str(exc)[:200])
        if key in self._hook_failures_reported:
            logger.debug("%s '%s' callback %s raised again: %s", surface, hook_name, callback_name, exc)
            return
        self._hook_failures_reported.add(key)
        logger.warning(
            "%s '%s' callback %s raised: %s (%s provides: %s; identical failures are logged at DEBUG from now on)",
            surface, hook_name, callback_name, exc, surface.lower(), ", ".join(sorted(kwargs)) or "no fields")

    def _run_hook_callback_bounded(
        self, hook_name: str, cb: Callable, kwargs: dict[str, Any], timeout: float
    ) -> Any:
        """Run one callback on a daemon worker with a wall-clock cap; ``_HOOK_SKIPPED`` when
        suppressed, still running for this call id, over the abandoned-worker cap, timed out
        (worker abandoned, never joined), or the worker could not be started. Exceptions
        propagate."""
        callback_name = getattr(cb, "__name__", repr(cb))
        # Suppression is a fact about the CALLBACK — a hung one must keep its back-off —
        # so that key stays coarse. The gate must instead tell CONCURRENT CALLS apart.
        suppression_key = (hook_name, id(cb))
        gate_key = (*suppression_key, _hook_call_identity(kwargs, hook_name))
        token = object()
        with self._hook_timeout_lock:
            suppressed_until = self._hook_timeout_suppressed_until.get(suppression_key)
            if (gate_key in self._hook_running_callbacks
                    or (suppressed_until is not None and suppressed_until > time.monotonic())):
                logger.warning(
                    "Hook '%s' callback %s skipped after previous "
                    "timeout or while still running", hook_name, callback_name)
                return _HOOK_SKIPPED
            # Workers abandoned on timeout still hold threads. Once the suppression window has
            # passed, a fresh call id may start a new worker (a hung guard must not fail every
            # later tool call closed until restart, #105223), but only up to a small cap per
            # callback — expiring the bookkeeping while the hung worker lives must not leak a
            # thread per call (#98382). At the cap the callback keeps being skipped (fail-closed
            # for pre_tool_call) until one of its workers finishes and releases its slot.
            abandoned = self._hook_abandoned.get(suppression_key)
            if abandoned and len(abandoned) >= _HOOK_MAX_ABANDONED_WORKERS:
                logger.warning(
                    "Hook '%s' callback %s (%s) skipped: %d abandoned worker(s) still running — "
                    "the plugin is hung; fix or disable it (retried when a worker finishes)",
                    hook_name, callback_name, getattr(cb, "__module__", "unknown plugin"), len(abandoned))
                return _HOOK_SKIPPED
            if suppressed_until is not None:
                self._hook_timeout_suppressed_until.pop(suppression_key, None)
            self._hook_running_callbacks[gate_key] = token

        context = contextvars.copy_context()
        done = threading.Event()
        outcome: dict[str, Any] = {}
        failure: dict[str, BaseException] = {}

        def _release_token() -> None:
            with self._hook_timeout_lock:
                if self._hook_running_callbacks.get(gate_key) is token:
                    self._hook_running_callbacks.pop(gate_key, None)
                    abandoned = self._hook_abandoned.get(suppression_key)
                    if abandoned is not None:
                        abandoned.discard(gate_key)
                        if not abandoned:
                            self._hook_abandoned.pop(suppression_key, None)

        def _runner() -> None:
            try:
                outcome["value"] = context.run(self._invoke_hook_callback, cb, kwargs)
            except BaseException as exc:
                failure["exc"] = exc
            finally:
                _release_token()
                done.set()

        thread = threading.Thread(target=_runner, name=f"hermes-hook-{callback_name}"[:40], daemon=True)
        try:
            thread.start()
        except RuntimeError as exc:
            _release_token()  # the runner's finally never runs when OS thread creation fails
            logger.warning(
                "Hook '%s' callback %s worker failed to start: %s — skipping",
                hook_name, callback_name, exc)
            return _HOOK_SKIPPED
        if not done.wait(timeout=timeout):  # do not join — that would reintroduce the hang
            with self._hook_timeout_lock:
                # See #6622.
                self._hook_timeout_suppressed_until[suppression_key] = (
                    time.monotonic() + self._hook_timeout_suppression_seconds)
                # The worker may have finished (and released its token) between the wait
                # expiring and this lock; recording it as abandoned then would block the
                # callback for that call id until reload with no thread behind it.
                if self._hook_running_callbacks.get(gate_key) is token:
                    self._hook_abandoned.setdefault(suppression_key, set()).add(gate_key)
            logger.warning(
                "Hook '%s' callback %s timed out after %gs — skipping", hook_name, callback_name, timeout)
            return _HOOK_SKIPPED
        if "exc" in failure:
            raise failure["exc"]
        return outcome.get("value")

    def _subscribe_event(self, owner: str, event: str, callback: Callable) -> None:
        """Add an owner-tagged event subscription in registration order."""
        if not callable(callback):
            raise TypeError("Event subscriber callback must be callable")
        with self._event_lock:
            self._subscriptions.setdefault(event, []).append(_EventSubscription(owner, callback))

    def _remove_plugin_subscriptions(self, owner: str) -> int:
        """Remove every subscription owned by *owner*; return the count. Queued envelopes re-check
        membership per callback, so this also cancels already-snapshotted deliveries.

        TODO(#64229): when the central plugin ownership ledger / registration handles land, route this
        owner-tagged bookkeeping through that ledger so per-plugin unload cancels event subscriptions
        alongside every other registration surface. This method is the integration seam.
        """
        removed = 0
        with self._event_lock:
            for event in list(self._subscriptions):
                entries = self._subscriptions[event]
                retained = [entry for entry in entries if entry.owner != owner]
                removed += len(entries) - len(retained)
                if retained:
                    self._subscriptions[event] = retained
                else:
                    del self._subscriptions[event]
        return removed

    def _ensure_event_worker_locked(self) -> None:
        worker = self._event_worker
        if worker is not None and worker.is_alive():
            return
        worker = threading.Thread(
            target=self._event_worker_loop, args=(self._event_queue,), name="hermes-plugin-events",
            daemon=True,
        )
        self._event_worker = worker
        worker.start()

    def _event_worker_loop(self, dispatch_queue: queue.Queue[Any]) -> None:
        while True:
            item = dispatch_queue.get()
            try:
                if item is _EVENT_WORKER_STOP:
                    return
                self._deliver_event(item)
            finally:
                if item is not _EVENT_WORKER_STOP:
                    self._mark_event_done(item.generation)
                dispatch_queue.task_done()

    def _mark_event_done(self, generation: int) -> None:
        with self._event_idle:
            pending = self._event_pending_by_generation.get(generation, 0)
            if pending > 0:
                self._event_pending_by_generation[generation] = pending - 1
            self._event_idle.notify_all()

    def _deliver_event(self, item: _QueuedPluginEvent) -> None:
        """Deliver one queued event on the host-owned worker thread."""
        with self._event_lock:
            if item.generation != self._event_generation:
                return
        previous_depth = getattr(self._emit_depth, "value", 0)
        self._emit_depth.value = item.depth
        try:
            for subscription in item.subscriptions:
                with self._event_lock:
                    if item.generation != self._event_generation:
                        break
                    # Owner unload may have removed this entry after the event was queued.
                    if not any(cur is subscription for cur in self._subscriptions.get(item.event, [])):
                        continue
                callback = subscription.callback
                try:
                    # Fresh deep copy per subscriber: no callback can mutate what the next sees.
                    resolve_plugin_command_result(
                        item.context.copy().run(callback, **copy.deepcopy(item.payload)))
                except (Exception, SystemExit) as exc:
                    # A subscriber that fails identically on every emit is reported once (#111922).
                    self._report_hook_failure(item.event, callback, item.payload, exc, surface="Event")
        finally:
            self._emit_depth.value = previous_depth

    def _wait_for_event_dispatch(self, timeout: float = 2.0) -> bool:
        """Wait for the current event generation to become idle (test helper)."""
        with self._event_idle:
            generation = self._event_generation
            return self._event_idle.wait_for(
                lambda: self._event_pending_by_generation.get(generation, 0) == 0, timeout=timeout)

    def _dispatch_event(self, event: str, payload: dict[str, Any]) -> int:
        """Queue *event* without blocking; return the subscriber count scheduled. Pending work is
        bounded per generation so a blocking subscriber costs one worker and later emits drop."""
        depth = getattr(self._emit_depth, "value", 0)
        if depth >= _EVENT_EMIT_DEPTH_CAP:
            logger.warning(
                "Event bus recursion cap (%d) exceeded while dispatching '%s' "
                "— dropping this emit to prevent an infinite loop", _EVENT_EMIT_DEPTH_CAP, event)
            return 0
        budget_msg = "Event bus pending budget (%d) exhausted while dispatching '%s' — dropping this emit"
        with self._event_lock:
            subscriptions = tuple(self._subscriptions.get(event, []))
            if not subscriptions:
                return 0
            generation = self._event_generation
            pending = self._event_pending_by_generation.get(generation, 0)
            if pending >= _EVENT_PENDING_CAP:
                logger.warning(budget_msg, _EVENT_PENDING_CAP, event)
                return 0
            item = _QueuedPluginEvent(
                event=event, payload=dict(payload), subscriptions=subscriptions, depth=depth + 1,
                generation=generation, context=contextvars.copy_context())
            try:
                self._event_queue.put_nowait(item)
            except queue.Full:
                logger.warning(budget_msg, _EVENT_PENDING_CAP, event)
                return 0
            self._event_pending_by_generation[generation] = pending + 1
            self._ensure_event_worker_locked()
            return len(subscriptions)

    def has_hook(self, hook_name: str) -> bool:
        """Return True when at least one callback is registered for a hook."""
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return False
        return bool(self._hooks.get(hook_name))

    def track_hook(self, context: Any, hook_name: str, callback: Callable, *, hold_stream: bool = False):
        """Register ``callback`` for ``hook_name`` on behalf of ``context`` (unknown names warn but are
        still stored). ``hold_stream=True`` on ``transform_llm_output`` declares the rewrite must reach
        the user first, so this profile's replies are not streamed (:meth:`holds_stream`)."""
        from hermes_cli.plugins import VALID_HOOKS

        handle = context._track_callback("hook", hook_name, callback, self._hooks, VALID_HOOKS)
        if hold_stream and hook_name != STREAM_HOLDING_HOOK:
            logger.warning("Plugin '%s' set hold_stream on '%s'; only %s rewrites the reply, ignored",
                           context.manifest.name, hook_name, STREAM_HOLDING_HOOK)
        else:
            handle.hold_stream = hold_stream
        return handle

    def holds_stream(self) -> bool:
        """True while a loaded plugin's ``transform_llm_output`` registration declared ``hold_stream``.
        Read under the discovery lock: a reload empties the ledger before plugins re-register, and a
        turn deciding in that window would stream text the plugin must rewrite first."""
        with self._discovery_lock:
            return any(r.hold_stream for owned in self._ownership_ledger.values() for r in owned)

    def held_transforms(self) -> tuple[Callable, ...]:
        """The ``transform_llm_output`` callbacks a turn that holds its stream must run, captured with
        the hold decision under the discovery lock; ``()`` when nothing holds. A reload during the turn
        empties ``_hooks`` before plugins re-register, so the held turn keeps the set it started with
        (:meth:`run_held_transforms`)."""
        with self._discovery_lock:
            if not any(r.hold_stream for owned in self._ownership_ledger.values() for r in owned):
                return ()
            return tuple(self._hooks.get(STREAM_HOLDING_HOOK, []))

    def run_held_transforms(self, callbacks: tuple[Callable, ...], **kwargs: Any) -> tuple[Optional[str], bool]:
        """Run a held turn's snapshotted transforms: ``(rewrite, failed)``. ``rewrite`` is the first
        non-empty string; ``failed`` is True when none came back and a callback raised or timed out,
        so the caller must not deliver the original."""
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return None, False
        kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
        failed: list = []
        for ret in self._run_hook_callbacks(STREAM_HOLDING_HOOK, callbacks, kwargs, failed):
            if isinstance(ret, str) and ret:
                return ret, False
        return None, bool(failed)

    async def ainvoke_hook(self, hook_name: str, **kwargs: Any) -> list[Any]:
        """:meth:`invoke_hook` for callers that are already on an event loop.

        Same payload narrowing, per-callback isolation and result contract. The difference is
        where an ``async def`` callback runs: here it is awaited on the caller's own loop, so a
        callback that awaits anything scheduled on that loop can make progress. Through the
        sync path it runs on a helper thread while the caller blocks in ``done.wait()`` — on the
        gateway that stalls the whole event loop for the callback's duration. Sync callbacks
        run inline. Bounded hooks keep ``plugins.hook_callback_timeout`` via ``asyncio.wait_for``
        (the coroutine is cancelled, not abandoned); a timed-out ``pre_tool_call`` fails closed.
        """
        from hermes_cli.plugins import _resolve_hook_callback_timeout
        if hook_name != "gateway_platform_event":
            kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
        results: list[Any] = []
        timeout = _resolve_hook_callback_timeout()
        use_timeout = _hook_uses_callback_timeout(hook_name, timeout)
        fail_closed = hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS
        for cb in self._hooks.get(hook_name, []):
            callback_name = getattr(cb, "__name__", repr(cb))
            try:
                ret = cb(**self._hook_callback_kwargs(cb, kwargs))
                if inspect.isawaitable(ret):
                    ret = await (asyncio.wait_for(ret, timeout) if use_timeout else ret)
                if ret is not None:
                    results.append(ret)
            except TimeoutError:
                logger.warning("Hook '%s' callback %s timed out after %.0fs", hook_name, callback_name, timeout)
                if fail_closed:  # policy hook: fail closed with a block directive
                    results.append({"action": "block", "message": _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE})
            except (Exception, SystemExit) as exc:
                # Same isolation + failure contract as the sync path (#111922 warn-once, #109624
                # a raising policy guard fails closed).
                self._report_hook_failure(hook_name, cb, kwargs, exc)
                if fail_closed:
                    results.append(_policy_error_block_directive(hook_name, cb, exc))
        return results

    def iter_hook_callbacks(self, hook_name: str) -> tuple[Callable, ...]:
        """Return a stable snapshot of callbacks registered for a hook."""
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return ()
        return tuple(self._hooks.get(hook_name, ()))

    def render_system_prompt_sections(
        self, session_info: Mapping[str, Any]
    ) -> list[RenderedPluginSystemPromptSection]:
        """Render all registered sections deterministically and fail open."""
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return []
        frozen_info = types.MappingProxyType(dict(session_info))
        rendered: list[RenderedPluginSystemPromptSection] = []
        total_chars = len(PLUGIN_SECTIONS_START) + len(PLUGIN_SECTIONS_END) + 2
        for _section_id, section in sorted(self._system_prompt_sections.items()):
            if len(rendered) >= MAX_SYSTEM_PROMPT_SECTIONS:
                logger.warning(
                    "Plugin system prompt section %s exceeded the section-count "
                    "budget (%d) and was skipped", section.id, MAX_SYSTEM_PROMPT_SECTIONS)
                continue
            text = self._render_prompt_section_text(section, frozen_info)
            if text is None:
                continue
            rendered_chars = len(format_system_prompt_section(section.id, text))
            if rendered:
                rendered_chars += 2  # canonical ``\n\n`` separator
            if total_chars + rendered_chars > MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS:
                logger.warning(
                    "Plugin system prompt section %s (%s) exceeded the aggregate "
                    "session budget (%d chars) and was skipped", section.id, section.plugin,
                    MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS)
                continue
            rendered.append(
                RenderedPluginSystemPromptSection(
                    id=section.id, content=text, position=section.position, plugin=section.plugin))
            total_chars += rendered_chars
            logger.info(
                "Session plugin prompt section: id=%s plugin=%s position=%s chars=%d", section.id,
                section.plugin, section.position, len(text))
        return rendered

    @staticmethod
    def _render_prompt_section_text(
        section: PluginSystemPromptSection, frozen_info: Mapping[str, Any]
    ) -> Optional[str]:
        """Evaluate one section; return its stripped text or None (with a warning) when skipped."""
        def _skip(detail: str, *args: Any) -> None:
            logger.warning(
                "Plugin system prompt section %s (%s) " + detail, section.id, section.plugin, *args)

        try:
            value = section.content(frozen_info) if callable(section.content) else section.content
        except (Exception, SystemExit) as exc:
            _skip("raised and was skipped: %s", exc)
            return None
        if not isinstance(value, str):
            _skip("returned %s, not str; skipped", type(value).__name__)
            return None
        text = value.strip()
        if not text:
            return None
        if PLUGIN_SECTIONS_START in text or PLUGIN_SECTIONS_END in text:
            _skip("contained a reserved persistence marker and was skipped")
            return None
        if len(text) > section.max_chars:
            _skip("exceeded max_chars (%d > %d) and was skipped", len(text), section.max_chars)
            return None
        return text

    def has_middleware(self, kind: str) -> bool:
        """Return True when at least one callback is registered for middleware."""
        from agent.safe_worker_policy import safe_worker_enabled

        if safe_worker_enabled():
            return False
        return bool(self._middleware.get(kind))

    def invoke_middleware(
        self, kind: str, *, _payload_key: Optional[str] = None, **kwargs: Any
    ) -> list[Any]:
        """Call middleware callbacks for *kind* (each isolated); return non-``None`` results.

        Request middleware passes ``_payload_key``: a dict returned under it becomes the payload the
        next callback sees, so rewrites compose, and each callback gets its own copy of the payload and
        of ``original_<key>`` so an in-place edit cannot leak.
        """
        from agent.safe_worker_policy import safe_worker_enabled
        from hermes_cli.middleware import _safe_copy

        if safe_worker_enabled():
            return []
        results: list[Any] = []
        for cb in self._middleware.get(kind, []):
            call_kwargs = kwargs
            if _payload_key:
                original_key = "original_" + _payload_key
                call_kwargs = {**kwargs, _payload_key: _safe_copy(kwargs[_payload_key]),
                               original_key: _safe_copy(kwargs[original_key])}
            try:
                ret = cb(**call_kwargs)
                if ret is not None:
                    results.append(ret)
                    if _payload_key and isinstance(ret, dict) and isinstance(ret.get(_payload_key), dict):
                        kwargs[_payload_key] = ret[_payload_key]
            except (Exception, SystemExit) as exc:
                # Runs once per tool call like a hook, so a mis-declared callback floods identically.
                self._report_hook_failure(kind, cb, kwargs, exc, surface="Middleware")
        return results