mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
fix(local-runtime): a crashed router's empty group is released before the restart backoff
The watcher noticed a crashed router, waited out the restart backoff (up to 60 s) and only then called stop_owned_group_survivors, which signals a group because its number is still in the registered set. When the crash left nothing in the group, that number was free for reuse for the whole backoff while still registered, so a stranger's group that got it would be sent TERM and then KILL. The watcher now releases an already-empty group as soon as it sees the crash; a group with survivors stays registered, and its number cannot be reused while they run. Test: a router stand-in that exits at once is no longer registered when the backoff starts; fails without the release.
This commit is contained in:
1 parent
b197ffca1e
commit
f84eb589ea
2 files changed
+31
-1
No files matched your search
@@ -27,7 +27,7 @@ from pathlib import Path
|
||||
from hermes_cli.local_runtime.binaries import runtimes_root
|
||||
from hermes_cli.local_runtime.endpoint import child_query
|
||||
from hermes_cli.local_runtime.processes import server_child_env, spawn_server
|
||||
from hermes_cli.process_identity import stop_owned_group_survivors
|
||||
from hermes_cli.process_identity import release_process_group_once_empty, stop_owned_group_survivors
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -269,6 +269,9 @@ class LlamaServerSupervisor:
|
||||
continue
|
||||
if self._stopping:
|
||||
return
|
||||
# An already-empty group is released now, not after the backoff: its number is free for
|
||||
# reuse meanwhile, and the survivor stop below would then signal a stranger's group.
|
||||
release_process_group_once_empty(proc.pid, proc=proc, timeout=0)
|
||||
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
|
||||
logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
|
||||
if self._stop_event.wait(backoff):
|
||||
|
||||
@@ -175,3 +175,30 @@ def test_a_crash_restart_stops_the_old_routers_survivors_and_releases_its_group(
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(old.pid, 9) # windows-footgun: ok — posix-marked test
|
||||
process_identity.release_process_groups([old.pid])
|
||||
|
||||
|
||||
def test_a_crashed_routers_empty_group_is_released_before_the_restart_backoff(monkeypatch, tmp_path):
|
||||
"""Kept registered through the backoff, the freed number could be reused by a stranger's group,
|
||||
which the restart's survivor stop would then TERM and KILL."""
|
||||
from hermes_cli import process_identity
|
||||
from hermes_cli.local_runtime import supervisor
|
||||
|
||||
monkeypatch.setattr(supervisor, "runtimes_root", lambda: tmp_path)
|
||||
sup = supervisor.LlamaServerSupervisor(tmp_path, tmp_path, port=59995)
|
||||
old = subprocess.Popen([sys.executable, "-c", "pass"], # detached-spawn: ok — test router stand-in; exits at once
|
||||
stdin=subprocess.DEVNULL, start_new_session=True)
|
||||
process_identity.own_process_groups([old.pid])
|
||||
registered_during_backoff = []
|
||||
|
||||
def backoff_wait(_seconds):
|
||||
registered_during_backoff.append(old.pid in process_identity._supervised_pgids)
|
||||
return True # stop() arrived during the backoff: ends the watch
|
||||
|
||||
try:
|
||||
old.wait(timeout=30) # the router crashed and left nothing in its group
|
||||
sup.proc = old
|
||||
monkeypatch.setattr(sup._stop_event, "wait", backoff_wait)
|
||||
sup._watch()
|
||||
assert registered_during_backoff == [False], "the empty group stayed registered through the backoff"
|
||||
finally:
|
||||
process_identity.release_process_groups([old.pid])
|
||||
Reference in new issue
Block a user