fix(local-runtime): a crashed router's empty group is released before the restart backoff

The watcher noticed a crashed router, waited out the restart backoff (up to 60 s) and only then
called stop_owned_group_survivors, which signals a group because its number is still in the
registered set. When the crash left nothing in the group, that number was free for reuse for the
whole backoff while still registered, so a stranger's group that got it would be sent TERM and then
KILL. The watcher now releases an already-empty group as soon as it sees the crash; a group with
survivors stays registered, and its number cannot be reused while they run.

Test: a router stand-in that exits at once is no longer registered when the backoff starts; fails
without the release.
This commit is contained in:
kshitijk4poor authored and kshitij committed 2026-10-29 16:58:09 +05:30
1 parent b197ffca1e
commit f84eb589ea
2 files changed
+31 -1

No files matched your search

+4 -1
View File
@@ -27,7 +27,7 @@ from pathlib import Path
from hermes_cli.local_runtime.binaries import runtimes_root
from hermes_cli.local_runtime.endpoint import child_query
from hermes_cli.local_runtime.processes import server_child_env, spawn_server
from hermes_cli.process_identity import stop_owned_group_survivors
from hermes_cli.process_identity import release_process_group_once_empty, stop_owned_group_survivors
logger = logging.getLogger(__name__)
@@ -269,6 +269,9 @@ class LlamaServerSupervisor:
continue
if self._stopping:
return
# An already-empty group is released now, not after the backoff: its number is free for
# reuse meanwhile, and the survivor stop below would then signal a stranger's group.
release_process_group_once_empty(proc.pid, proc=proc, timeout=0)
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
if self._stop_event.wait(backoff):
@@ -175,3 +175,30 @@ def test_a_crash_restart_stops_the_old_routers_survivors_and_releases_its_group(
with contextlib.suppress(ProcessLookupError):
os.killpg(old.pid, 9) # windows-footgun: ok — posix-marked test
process_identity.release_process_groups([old.pid])
def test_a_crashed_routers_empty_group_is_released_before_the_restart_backoff(monkeypatch, tmp_path):
"""Kept registered through the backoff, the freed number could be reused by a stranger's group,
which the restart's survivor stop would then TERM and KILL."""
from hermes_cli import process_identity
from hermes_cli.local_runtime import supervisor
monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59995)
old = subprocess.Popen([sys.executable, "-c", "pass"], # detached-spawn: ok — test router stand-in; exits at once
stdin=subprocess.DEVNULL, start_new_session=True)
process_identity.own_process_groups([old.pid])
registered_during_backoff = []
def backoff_wait(_seconds):
registered_during_backoff.append(old.pid in process_identity._supervised_pgids)
return True # stop() arrived during the backoff: ends the watch
try:
old.wait(timeout=30) # the router crashed and left nothing in its group
sup.proc = old
monkeypatch.setattr(sup._stop_event, "wait", backoff_wait)
sup._watch()
assert registered_during_backoff == [False], "the empty group stayed registered through the backoff"
finally:
process_identity.release_process_groups([old.pid])