From 8bfa0160f888b2d696fefcfafd0c12a7bc8c2d1f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:01:01 -0700 Subject: [PATCH] fix(agent): drop the unscored-run warning once newer evidence is recorded Review finding "Warning names an older run": the remembered unscored run (`pytest || true`) was cleared only by record_terminal_result. After it, a failed `hermes verify` still left the nudge calling the older command the "last check run". The clear moves to _insert_evidence, the boundary both recording paths share. `hermes verify` usually records from its own process, though, so the memory also keeps when the run happened and the nudge shows the warning only when no evidence was recorded after it. --- agent/verification_evidence.py | 25 +++++++++++++++++-------- agent/verification_stop.py | 4 +++- tests/agent/test_verification_stop.py | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/agent/verification_evidence.py b/agent/verification_evidence.py index 83e3de3860..17931f8d3e 100644 --- a/agent/verification_evidence.py +++ b/agent/verification_evidence.py @@ -25,9 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100 _MAX_TOTAL_UNREFERENCED_EVENTS = 10_000 _AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-") _VERIFY_SCHEMA_VERSION = 1 -# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by -# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways. -_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {} +# Latest verify-command run per (session, root) whose own exit status the shell hid, with the +# time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for +# long-lived gateways. +_UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {} _MAX_UNATTRIBUTED_RUNS = 256 _INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"} @@ -582,20 +583,27 @@ def record_terminal_result( pipeline_status=pipeline_status, ) if evidence is not None: - _UNATTRIBUTED_RUNS.pop(key, None) return _insert_evidence(evidence) verify_commands = list(facts.get("verifyCommands") or []) if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False): _UNATTRIBUTED_RUNS.pop(key, None) - _UNATTRIBUTED_RUNS[key] = command + _UNATTRIBUTED_RUNS[key] = (command, _utc_now()) while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS: _UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS))) return None -def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]: - """The latest verify-command run in ``root`` whose own exit status could not be read.""" - return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root))) +def last_unattributed_run( + *, session_id: str | None, root: str | None, since: str | None = None, +) -> Optional[str]: + """The latest verify-command run in ``root`` whose own exit status could not be read, + unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records + from its own process, so this process's memory alone cannot know about that run. + """ + run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root))) + if run is None or (since and since >= run[1]): + return None + return run[0] def record_verify_run( @@ -624,6 +632,7 @@ def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]: """Insert a classified evidence row and repoint the workspace state.""" created_at = _utc_now() e = evidence + _UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None) with _DB_LOCK, _transaction() as conn: cur = conn.execute( "INSERT INTO verification_events(" diff --git a/agent/verification_stop.py b/agent/verification_stop.py index 74b4a1c9f4..20af7a65d6 100644 --- a/agent/verification_stop.py +++ b/agent/verification_stop.py @@ -181,7 +181,9 @@ def build_verify_on_stop_nudge( status, facts = snapshot if str(status.get("status") or "unverified") == "passed": return None - unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root")) + unattributed = last_unattributed_run( + session_id=status.get("session_id"), root=status.get("root"), + since=(status.get("evidence") or {}).get("created_at")) run_warning = _UNATTRIBUTED_RUN_WARNING.format( command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else "" verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()] diff --git a/tests/agent/test_verification_stop.py b/tests/agent/test_verification_stop.py index 82ca2eca1a..8b4a6b21de 100644 --- a/tests/agent/test_verification_stop.py +++ b/tests/agent/test_verification_stop.py @@ -1,4 +1,5 @@ import json +import subprocess import sys import tempfile from pathlib import Path @@ -189,6 +190,24 @@ def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path, assert "Your last check run (`pnpm test || true`) did not count" in hidden assert "did not count" not in scored +def test_nudge_drops_the_unscored_run_warning_once_hermes_verify_records(tmp_path, monkeypatch): + """``hermes verify`` records from its own process: its newer failure is the latest check.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + _node_project(tmp_path) + changed = str(tmp_path / "src" / "app.ts") + mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed]) + record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0) + + subprocess.run( + [sys.executable, "-c", "import sys; from agent.verification_evidence import record_verify_run; " + "record_verify_run(root=sys.argv[1], session_id='s1', ok=False)", str(tmp_path)], + cwd=Path(__file__).resolve().parents[2], check=True, + ) + nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed]) + + assert nudge is not None + assert "hermes verify" in nudge and "did not count" not in nudge + # --------------------------------------------------------------------------- # Fix C: documentation/prose edits carry no verifiable behavior and must never # trip the nudge, even on an unverified workspace.