fix(agent): drop the unscored-run warning once newer evidence is recorded

Review finding "Warning names an older run": the remembered unscored run
(`pytest || true`) was cleared only by record_terminal_result. After it, a
failed `hermes verify` still left the nudge calling the older command the
"last check run".

The clear moves to _insert_evidence, the boundary both recording paths
share. `hermes verify` usually records from its own process, though, so the
memory also keeps when the run happened and the nudge shows the warning
only when no evidence was recorded after it.
This commit is contained in:
Teknium committed 2026-10-24 18:48:01 -07:00
1 parent c98e8dc63a
commit 8bfa0160f8
3 files changed
+39 -9

No files matched your search

+17 -8
View File
@@ -25,9 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100
_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000
_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")
_VERIFY_SCHEMA_VERSION = 1
# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by
# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.
_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}
# Latest verify-command run per (session, root) whose own exit status the shell hid, with the
# time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for
# long-lived gateways.
_UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {}
_MAX_UNATTRIBUTED_RUNS = 256
_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}
@@ -582,20 +583,27 @@ def record_terminal_result(
pipeline_status=pipeline_status,
)
if evidence is not None:
_UNATTRIBUTED_RUNS.pop(key, None)
return _insert_evidence(evidence)
verify_commands = list(facts.get("verifyCommands") or [])
if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):
_UNATTRIBUTED_RUNS.pop(key, None)
_UNATTRIBUTED_RUNS[key] = command
_UNATTRIBUTED_RUNS[key] = (command, _utc_now())
while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:
_UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))
return None
def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:
"""The latest verify-command run in ``root`` whose own exit status could not be read."""
return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
def last_unattributed_run(
*, session_id: str | None, root: str | None, since: str | None = None,
) -> Optional[str]:
"""The latest verify-command run in ``root`` whose own exit status could not be read,
unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records
from its own process, so this process's memory alone cannot know about that run.
"""
run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
if run is None or (since and since >= run[1]):
return None
return run[0]
def record_verify_run(
@@ -624,6 +632,7 @@ def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:
"""Insert a classified evidence row and repoint the workspace state."""
created_at = _utc_now()
e = evidence
_UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None)
with _DB_LOCK, _transaction() as conn:
cur = conn.execute(
"INSERT INTO verification_events("
+3 -1
View File
@@ -181,7 +181,9 @@ def build_verify_on_stop_nudge(
status, facts = snapshot
if str(status.get("status") or "unverified") == "passed":
return None
unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))
unattributed = last_unattributed_run(
session_id=status.get("session_id"), root=status.get("root"),
since=(status.get("evidence") or {}).get("created_at"))
run_warning = _UNATTRIBUTED_RUN_WARNING.format(
command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""
verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]
+19
View File
@@ -1,4 +1,5 @@
import json
import subprocess
import sys
import tempfile
from pathlib import Path
@@ -189,6 +190,24 @@ def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path,
assert "Your last check run (`pnpm test || true`) did not count" in hidden
assert "did not count" not in scored
def test_nudge_drops_the_unscored_run_warning_once_hermes_verify_records(tmp_path, monkeypatch):
"""``hermes verify`` records from its own process: its newer failure is the latest check."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
_node_project(tmp_path)
changed = str(tmp_path / "src" / "app.ts")
mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])
record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)
subprocess.run(
[sys.executable, "-c", "import sys; from agent.verification_evidence import record_verify_run; "
"record_verify_run(root=sys.argv[1], session_id='s1', ok=False)", str(tmp_path)],
cwd=Path(__file__).resolve().parents[2], check=True,
)
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
assert nudge is not None
assert "hermes verify" in nudge and "did not count" not in nudge
# ---------------------------------------------------------------------------
# Fix C: documentation/prose edits carry no verifiable behavior and must never
# trip the nudge, even on an unverified workspace.