mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
Twenty loops across 17 files hand-wrote the same thing:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if condition(): return ...
time.sleep(interval)
with small drifts between copies: some re-checked once after the loop,
some did not, one slept before its first check, and update_cmd_windows
kept a private _poll_until. A copy without the final check can miss a
condition that becomes true during the last sleep.
agent/deadline.py (already the home of clamp_timeout / run_bounded_*)
gains poll_until(predicate, timeout, interval): checks at once, cuts the
last sleep to the time left, checks again at the deadline, and returns
the predicate's last value so callers that wait for a pid / row /
identity get it back directly. timeout <= 0 is one check.
Converted: outbound_webhooks.flush, cron delivery wait and jobs flock,
host_attach channel wait, browser profile unlock, data-cleanup drain,
gateway dbus/bus waits, launchd fresh-pid and supervision waits,
supervised restart, gateway_migrate live-gateway wait, Windows
drain/absent waits, kanban init lock, local-runtime boot wait,
update_cmd_windows (private helper deleted), bot desktop start/stop
waits, TUI browser connect.
Left as is: loops whose body does more than check (progress callbacks,
interrupt handling, two distinct exits) and one site whose blind except
the health gate would reject once moved.
Behaviour changes, all small:
- loops that stopped at the deadline without re-checking now make one
more check there;
- loops that already re-checked after a full last sleep now cut that
sleep to the time left, so the check lands at the deadline (up to one
interval sooner);
- a timeout <= 0 now means one check (some loops made none);
- the local-runtime boot wait checks before its first sleep instead of
after it;
- _wait_for_user_dbus_socket also exports the user systemd env when the
socket appears on that final check (before, that path returned True
without it).
update_cmd_windows is imported by update_cmd on every OS, so the
fresh-source-install acceptance fixture (which keeps agent/ out of the
update path) now copies the stdlib-only agent/deadline.py too.
Tests: 4 pins for poll_until (two red against the classic
no-final-check variant), plus the existing suites for every touched
module.
169 lines
8.0 KiB
Python
169 lines
8.0 KiB
Python
"""Home-scoped runtime coordination for an explicitly confirmed data wipe."""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import ExitStack, contextmanager
|
|
from pathlib import Path
|
|
|
|
from agent.deadline import poll_until
|
|
|
|
|
|
_OWNER_STOP_HINTS = {
|
|
"systemd": "stop its systemd unit with systemctl before retrying",
|
|
"launchd": "stop its LaunchAgent with launchctl before retrying",
|
|
"desktop": "quit the owning desktop app before retrying",
|
|
"external": "stop the gateway through its external supervisor before retrying",
|
|
}
|
|
|
|
|
|
def manual_gateway_identity(home: Path) -> dict | None:
|
|
"""The ``identify`` answer of the manually launched gateway owning ``home``, None when none runs.
|
|
|
|
Raises RuntimeError for a gateway this client may not drain: a supervisor (systemd, launchd,
|
|
Desktop, ...) relaunches it right after a pause, so it would take the profile back.
|
|
"""
|
|
from gateway.control_socket import identify_gateway, launch_supervisor
|
|
from gateway.status import get_running_pid_identity_strict
|
|
|
|
identity = identify_gateway(home)
|
|
if identity is None:
|
|
if get_running_pid_identity_strict(home / "gateway.pid") is not None:
|
|
raise RuntimeError(f"gateway has no usable control socket for {home}; stop it through its owner and retry")
|
|
return None
|
|
if not identity.get("hermes_home") or Path(identity["hermes_home"]).resolve() != home:
|
|
raise RuntimeError(f"gateway control identity does not match {home}")
|
|
owner = launch_supervisor(identity)
|
|
if owner != "manual":
|
|
hint = _OWNER_STOP_HINTS.get(owner or "", "stop it through its supervisor before retrying")
|
|
raise RuntimeError(f"{owner or 'unknown'} gateway still owns {home}; {hint}")
|
|
return identity
|
|
|
|
|
|
def pause_acked(response: dict | None, identity: dict | None) -> bool:
|
|
"""True when the identified gateway itself accepted (or is already running) the drain."""
|
|
return bool(response and identity is not None and response.get("pid") == identity.get("pid")
|
|
and (response.get("pausing") or response.get("already_stopping")))
|
|
|
|
|
|
def _drain_manual_gateway(home: Path) -> None:
|
|
from gateway.control_socket import identify_gateway, pause_gateway_for_update
|
|
from gateway.status import get_process_start_time, get_running_pid_identity_strict, _pid_exists
|
|
|
|
identity = manual_gateway_identity(home)
|
|
if identity is None:
|
|
return
|
|
if identity.get("served_profiles"):
|
|
raise RuntimeError("gateway serves multiple profiles; stop it explicitly before deleting one profile's data")
|
|
pid = identity.get("pid")
|
|
start = identity.get("start_time")
|
|
if type(pid) is not int or pid <= 0 or start is None or get_process_start_time(pid) != start:
|
|
raise RuntimeError(f"gateway process identity is unverified for {home}")
|
|
response = pause_gateway_for_update(home)
|
|
if not pause_acked(response, identity):
|
|
raise RuntimeError(f"gateway refused to drain for {home}")
|
|
def _gone() -> bool:
|
|
current = get_process_start_time(pid)
|
|
original_gone = not _pid_exists(pid) or (current is not None and current != start)
|
|
return original_gone and identify_gateway(home) is None and get_running_pid_identity_strict(home / "gateway.pid") is None
|
|
|
|
if not poll_until(_gone, min(120.0, max(5.0, float(response.get("drain_timeout", 30.0)) + 5.0)), 0.05):
|
|
raise RuntimeError(f"gateway did not exit for {home}; no data removed")
|
|
|
|
|
|
def _refuse_backend_writers(home: Path) -> None:
|
|
from hermes_constants import get_default_hermes_root
|
|
from hermes_cli.process_identity import LEDGER_FILENAME, _read_ledger, entry_still_running
|
|
|
|
root = get_default_hermes_root(home=home).resolve()
|
|
rows = _read_ledger(root / LEDGER_FILENAME)
|
|
if rows is None:
|
|
raise RuntimeError(f"cannot read backend ownership: {root / LEDGER_FILENAME}")
|
|
for row in rows:
|
|
if row.get("purpose") not in {"serve", "dashboard"}:
|
|
continue
|
|
# The recorded profile is the initial UI selection, not the backend's
|
|
# write scope: it can serve any profile under this machine root.
|
|
pid = row.get("pid")
|
|
if type(pid) is not int or pid <= 0:
|
|
raise RuntimeError("backend ownership contains an invalid PID")
|
|
if entry_still_running(row):
|
|
raise RuntimeError(f"backend PID {pid} still owns {home}; close its desktop/dashboard before retrying")
|
|
|
|
|
|
def _refuse_cron_writers(home: Path) -> None:
|
|
from contextlib import closing
|
|
import sqlite3
|
|
from gateway.status import get_process_start_time, _pid_exists
|
|
|
|
path = home / "cron" / "executions.db"
|
|
try:
|
|
path.stat()
|
|
except FileNotFoundError:
|
|
return
|
|
# Importing cron initializes scheduler/config defaults. A deletion probe
|
|
# must only read the existing ledger, not recreate the data being removed.
|
|
try:
|
|
with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.25)) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
rows = conn.execute("SELECT * FROM executions WHERE status IN ('claimed','running')").fetchall()
|
|
except sqlite3.Error as exc:
|
|
raise RuntimeError(f"cannot inspect cron owners in {path}: {exc}") from exc
|
|
for raw in rows:
|
|
row = dict(raw)
|
|
pid = row["pid"]
|
|
if row.get("handoff_pending"):
|
|
raise RuntimeError(f"cron execution {row['id']} is being handed to a worker; wait for it to finish")
|
|
if not _pid_exists(pid):
|
|
continue
|
|
current = get_process_start_time(pid)
|
|
recorded = row.get("process_started_at")
|
|
if current is None or recorded is None or current == recorded:
|
|
raise RuntimeError(f"cron execution {row['id']} still owns {home} (PID {pid}); stop it before retrying")
|
|
|
|
|
|
def _refuse_multiplexer(home: Path) -> None:
|
|
from hermes_constants import get_default_hermes_root
|
|
from gateway.control_socket import identify_gateway
|
|
from gateway.status import get_running_pid_identity_strict
|
|
|
|
root = get_default_hermes_root(home=home).resolve()
|
|
if home == root:
|
|
return
|
|
identity = identify_gateway(root)
|
|
if identity is not None:
|
|
if home.name in identity.get("served_profiles", []):
|
|
raise RuntimeError(f"the default gateway serves {home}; stop the multiplexer explicitly before retrying")
|
|
elif get_running_pid_identity_strict(root / "gateway.pid") is not None:
|
|
raise RuntimeError("the default gateway's profile scope is unverified; stop it explicitly before retrying")
|
|
|
|
|
|
@contextmanager
|
|
def quiescent_home(home: Path):
|
|
"""Hold existing writer locks through deletion; do not remove their inodes."""
|
|
from gateway.status import _release_file_lock, _try_acquire_file_lock
|
|
from hermes_cli.active_sessions import _FileLock, _prune_dead, _read_entries
|
|
from hermes_cli.runtime_state import _lock
|
|
from tools.checkpoint_pruning import store_lock
|
|
|
|
home = home.resolve()
|
|
_refuse_backend_writers(home)
|
|
_refuse_multiplexer(home)
|
|
_drain_manual_gateway(home)
|
|
with ExitStack() as stack:
|
|
stack.enter_context(store_lock(home / "checkpoints"))
|
|
gateway_lock = stack.enter_context((home / "gateway.lock").open("a+", encoding="utf-8"))
|
|
if not _try_acquire_file_lock(gateway_lock):
|
|
raise RuntimeError(f"gateway is still active for {home}")
|
|
stack.callback(_release_file_lock, gateway_lock)
|
|
stack.enter_context(_FileLock(home / "runtime" / "active_sessions.lock"))
|
|
sessions = _prune_dead(_read_entries(home / "runtime" / "active_sessions.json", strict=True), strict=True)
|
|
if sessions:
|
|
pids = sorted({entry["pid"] for entry in sessions})
|
|
raise RuntimeError(f"live chat sessions still own {home} (PIDs {pids}); close them before retrying")
|
|
for path in (home / ".backup.lock", home / "cron" / ".tick.lock"):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
handle = stack.enter_context(path.open("a+b"))
|
|
if not _lock(handle.fileno(), wait=False):
|
|
raise RuntimeError(f"an active writer still owns {path}")
|
|
_refuse_cron_writers(home)
|
|
yield
|