Files
hermes-agent/hermes_cli/data_cleanup_holders.py
kshitijk4poor de5f56c92a refactor(deadline): one poll_until for check-until-deadline loops
Twenty loops across 17 files hand-wrote the same thing:

    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        if condition(): return ...
        time.sleep(interval)

with small drifts between copies: some re-checked once after the loop,
some did not, one slept before its first check, and update_cmd_windows
kept a private _poll_until. A copy without the final check can miss a
condition that becomes true during the last sleep.

agent/deadline.py (already the home of clamp_timeout / run_bounded_*)
gains poll_until(predicate, timeout, interval): checks at once, cuts the
last sleep to the time left, checks again at the deadline, and returns
the predicate's last value so callers that wait for a pid / row /
identity get it back directly. timeout <= 0 is one check.

Converted: outbound_webhooks.flush, cron delivery wait and jobs flock,
host_attach channel wait, browser profile unlock, data-cleanup drain,
gateway dbus/bus waits, launchd fresh-pid and supervision waits,
supervised restart, gateway_migrate live-gateway wait, Windows
drain/absent waits, kanban init lock, local-runtime boot wait,
update_cmd_windows (private helper deleted), bot desktop start/stop
waits, TUI browser connect.

Left as is: loops whose body does more than check (progress callbacks,
interrupt handling, two distinct exits) and one site whose blind except
the health gate would reject once moved.

Behaviour changes, all small:
- loops that stopped at the deadline without re-checking now make one
  more check there;
- loops that already re-checked after a full last sleep now cut that
  sleep to the time left, so the check lands at the deadline (up to one
  interval sooner);
- a timeout <= 0 now means one check (some loops made none);
- the local-runtime boot wait checks before its first sleep instead of
  after it;
- _wait_for_user_dbus_socket also exports the user systemd env when the
  socket appears on that final check (before, that path returned True
  without it).

update_cmd_windows is imported by update_cmd on every OS, so the
fresh-source-install acceptance fixture (which keeps agent/ out of the
update path) now copies the stdlib-only agent/deadline.py too.

Tests: 4 pins for poll_until (two red against the classic
no-final-check variant), plus the existing suites for every touched
module.
2026-10-27 16:30:14 +05:30

169 lines
8.0 KiB
Python

"""Home-scoped runtime coordination for an explicitly confirmed data wipe."""
from __future__ import annotations
from contextlib import ExitStack, contextmanager
from pathlib import Path
from agent.deadline import poll_until
_OWNER_STOP_HINTS = {
"systemd": "stop its systemd unit with systemctl before retrying",
"launchd": "stop its LaunchAgent with launchctl before retrying",
"desktop": "quit the owning desktop app before retrying",
"external": "stop the gateway through its external supervisor before retrying",
}
def manual_gateway_identity(home: Path) -> dict | None:
"""The ``identify`` answer of the manually launched gateway owning ``home``, None when none runs.
Raises RuntimeError for a gateway this client may not drain: a supervisor (systemd, launchd,
Desktop, ...) relaunches it right after a pause, so it would take the profile back.
"""
from gateway.control_socket import identify_gateway, launch_supervisor
from gateway.status import get_running_pid_identity_strict
identity = identify_gateway(home)
if identity is None:
if get_running_pid_identity_strict(home / "gateway.pid") is not None:
raise RuntimeError(f"gateway has no usable control socket for {home}; stop it through its owner and retry")
return None
if not identity.get("hermes_home") or Path(identity["hermes_home"]).resolve() != home:
raise RuntimeError(f"gateway control identity does not match {home}")
owner = launch_supervisor(identity)
if owner != "manual":
hint = _OWNER_STOP_HINTS.get(owner or "", "stop it through its supervisor before retrying")
raise RuntimeError(f"{owner or 'unknown'} gateway still owns {home}; {hint}")
return identity
def pause_acked(response: dict | None, identity: dict | None) -> bool:
"""True when the identified gateway itself accepted (or is already running) the drain."""
return bool(response and identity is not None and response.get("pid") == identity.get("pid")
and (response.get("pausing") or response.get("already_stopping")))
def _drain_manual_gateway(home: Path) -> None:
from gateway.control_socket import identify_gateway, pause_gateway_for_update
from gateway.status import get_process_start_time, get_running_pid_identity_strict, _pid_exists
identity = manual_gateway_identity(home)
if identity is None:
return
if identity.get("served_profiles"):
raise RuntimeError("gateway serves multiple profiles; stop it explicitly before deleting one profile's data")
pid = identity.get("pid")
start = identity.get("start_time")
if type(pid) is not int or pid <= 0 or start is None or get_process_start_time(pid) != start:
raise RuntimeError(f"gateway process identity is unverified for {home}")
response = pause_gateway_for_update(home)
if not pause_acked(response, identity):
raise RuntimeError(f"gateway refused to drain for {home}")
def _gone() -> bool:
current = get_process_start_time(pid)
original_gone = not _pid_exists(pid) or (current is not None and current != start)
return original_gone and identify_gateway(home) is None and get_running_pid_identity_strict(home / "gateway.pid") is None
if not poll_until(_gone, min(120.0, max(5.0, float(response.get("drain_timeout", 30.0)) + 5.0)), 0.05):
raise RuntimeError(f"gateway did not exit for {home}; no data removed")
def _refuse_backend_writers(home: Path) -> None:
from hermes_constants import get_default_hermes_root
from hermes_cli.process_identity import LEDGER_FILENAME, _read_ledger, entry_still_running
root = get_default_hermes_root(home=home).resolve()
rows = _read_ledger(root / LEDGER_FILENAME)
if rows is None:
raise RuntimeError(f"cannot read backend ownership: {root / LEDGER_FILENAME}")
for row in rows:
if row.get("purpose") not in {"serve", "dashboard"}:
continue
# The recorded profile is the initial UI selection, not the backend's
# write scope: it can serve any profile under this machine root.
pid = row.get("pid")
if type(pid) is not int or pid <= 0:
raise RuntimeError("backend ownership contains an invalid PID")
if entry_still_running(row):
raise RuntimeError(f"backend PID {pid} still owns {home}; close its desktop/dashboard before retrying")
def _refuse_cron_writers(home: Path) -> None:
from contextlib import closing
import sqlite3
from gateway.status import get_process_start_time, _pid_exists
path = home / "cron" / "executions.db"
try:
path.stat()
except FileNotFoundError:
return
# Importing cron initializes scheduler/config defaults. A deletion probe
# must only read the existing ledger, not recreate the data being removed.
try:
with closing(sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=0.25)) as conn:
conn.row_factory = sqlite3.Row
rows = conn.execute("SELECT * FROM executions WHERE status IN ('claimed','running')").fetchall()
except sqlite3.Error as exc:
raise RuntimeError(f"cannot inspect cron owners in {path}: {exc}") from exc
for raw in rows:
row = dict(raw)
pid = row["pid"]
if row.get("handoff_pending"):
raise RuntimeError(f"cron execution {row['id']} is being handed to a worker; wait for it to finish")
if not _pid_exists(pid):
continue
current = get_process_start_time(pid)
recorded = row.get("process_started_at")
if current is None or recorded is None or current == recorded:
raise RuntimeError(f"cron execution {row['id']} still owns {home} (PID {pid}); stop it before retrying")
def _refuse_multiplexer(home: Path) -> None:
from hermes_constants import get_default_hermes_root
from gateway.control_socket import identify_gateway
from gateway.status import get_running_pid_identity_strict
root = get_default_hermes_root(home=home).resolve()
if home == root:
return
identity = identify_gateway(root)
if identity is not None:
if home.name in identity.get("served_profiles", []):
raise RuntimeError(f"the default gateway serves {home}; stop the multiplexer explicitly before retrying")
elif get_running_pid_identity_strict(root / "gateway.pid") is not None:
raise RuntimeError("the default gateway's profile scope is unverified; stop it explicitly before retrying")
@contextmanager
def quiescent_home(home: Path):
"""Hold existing writer locks through deletion; do not remove their inodes."""
from gateway.status import _release_file_lock, _try_acquire_file_lock
from hermes_cli.active_sessions import _FileLock, _prune_dead, _read_entries
from hermes_cli.runtime_state import _lock
from tools.checkpoint_pruning import store_lock
home = home.resolve()
_refuse_backend_writers(home)
_refuse_multiplexer(home)
_drain_manual_gateway(home)
with ExitStack() as stack:
stack.enter_context(store_lock(home / "checkpoints"))
gateway_lock = stack.enter_context((home / "gateway.lock").open("a+", encoding="utf-8"))
if not _try_acquire_file_lock(gateway_lock):
raise RuntimeError(f"gateway is still active for {home}")
stack.callback(_release_file_lock, gateway_lock)
stack.enter_context(_FileLock(home / "runtime" / "active_sessions.lock"))
sessions = _prune_dead(_read_entries(home / "runtime" / "active_sessions.json", strict=True), strict=True)
if sessions:
pids = sorted({entry["pid"] for entry in sessions})
raise RuntimeError(f"live chat sessions still own {home} (PIDs {pids}); close them before retrying")
for path in (home / ".backup.lock", home / "cron" / ".tick.lock"):
path.parent.mkdir(parents=True, exist_ok=True)
handle = stack.enter_context(path.open("a+b"))
if not _lock(handle.fileno(), wait=False):
raise RuntimeError(f"an active writer still owns {path}")
_refuse_cron_writers(home)
yield