mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-06 05:17:15 +00:00
fix(agent): drop the unscored-run warning once newer evidence is recorded
Review finding "Warning names an older run": the remembered unscored run (`pytest || true`) was cleared only by record_terminal_result. After it, a failed `hermes verify` still left the nudge calling the older command the "last check run". The clear moves to _insert_evidence, the boundary both recording paths share. `hermes verify` usually records from its own process, though, so the memory also keeps when the run happened and the nudge shows the warning only when no evidence was recorded after it.
This commit is contained in:
1 parent
c98e8dc63a
commit
8bfa0160f8
3 files changed
+39
-9
No files matched your search
@@ -25,9 +25,10 @@ _MAX_EVENTS_PER_SESSION_ROOT = 100
|
||||
_MAX_TOTAL_UNREFERENCED_EVENTS = 10_000
|
||||
_AD_HOC_SCRIPT_NAME_PREFIXES = ("hermes-verify-", "hermes-ad-hoc-")
|
||||
_VERIFY_SCHEMA_VERSION = 1
|
||||
# Latest verify-command run per (session, root) whose own exit status the shell hid. Read by
|
||||
# the stop nudge in this same process, so it stays in memory; bounded for long-lived gateways.
|
||||
_UNATTRIBUTED_RUNS: dict[tuple[str, str], str] = {}
|
||||
# Latest verify-command run per (session, root) whose own exit status the shell hid, with the
|
||||
# time it ran. Read by the stop nudge in this same process, so it stays in memory; bounded for
|
||||
# long-lived gateways.
|
||||
_UNATTRIBUTED_RUNS: dict[tuple[str, str], tuple[str, str]] = {}
|
||||
_MAX_UNATTRIBUTED_RUNS = 256
|
||||
|
||||
_INTERPRETERS = {"python", "python3", "py", "node", "bash", "sh", "ruby", "perl"}
|
||||
@@ -582,20 +583,27 @@ def record_terminal_result(
|
||||
pipeline_status=pipeline_status,
|
||||
)
|
||||
if evidence is not None:
|
||||
_UNATTRIBUTED_RUNS.pop(key, None)
|
||||
return _insert_evidence(evidence)
|
||||
verify_commands = list(facts.get("verifyCommands") or [])
|
||||
if _find_canonical_match(command, verify_commands, int(exit_code), attributed=False):
|
||||
_UNATTRIBUTED_RUNS.pop(key, None)
|
||||
_UNATTRIBUTED_RUNS[key] = command
|
||||
_UNATTRIBUTED_RUNS[key] = (command, _utc_now())
|
||||
while len(_UNATTRIBUTED_RUNS) > _MAX_UNATTRIBUTED_RUNS:
|
||||
_UNATTRIBUTED_RUNS.pop(next(iter(_UNATTRIBUTED_RUNS)))
|
||||
return None
|
||||
|
||||
|
||||
def last_unattributed_run(*, session_id: str | None, root: str | None) -> Optional[str]:
|
||||
"""The latest verify-command run in ``root`` whose own exit status could not be read."""
|
||||
return _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
|
||||
def last_unattributed_run(
|
||||
*, session_id: str | None, root: str | None, since: str | None = None,
|
||||
) -> Optional[str]:
|
||||
"""The latest verify-command run in ``root`` whose own exit status could not be read,
|
||||
unless evidence recorded at ``since`` came after it. ``hermes verify`` usually records
|
||||
from its own process, so this process's memory alone cannot know about that run.
|
||||
"""
|
||||
run = _UNATTRIBUTED_RUNS.get((str(session_id or "default"), str(root)))
|
||||
if run is None or (since and since >= run[1]):
|
||||
return None
|
||||
return run[0]
|
||||
|
||||
|
||||
def record_verify_run(
|
||||
@@ -624,6 +632,7 @@ def _insert_evidence(evidence: VerificationEvidence) -> dict[str, Any]:
|
||||
"""Insert a classified evidence row and repoint the workspace state."""
|
||||
created_at = _utc_now()
|
||||
e = evidence
|
||||
_UNATTRIBUTED_RUNS.pop((e.session_id, e.root), None)
|
||||
with _DB_LOCK, _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO verification_events("
|
||||
|
||||
@@ -181,7 +181,9 @@ def build_verify_on_stop_nudge(
|
||||
status, facts = snapshot
|
||||
if str(status.get("status") or "unverified") == "passed":
|
||||
return None
|
||||
unattributed = last_unattributed_run(session_id=status.get("session_id"), root=status.get("root"))
|
||||
unattributed = last_unattributed_run(
|
||||
session_id=status.get("session_id"), root=status.get("root"),
|
||||
since=(status.get("evidence") or {}).get("created_at"))
|
||||
run_warning = _UNATTRIBUTED_RUN_WARNING.format(
|
||||
command=elide(unattributed, _MAX_UNATTRIBUTED_COMMAND_CHARS)) if unattributed else ""
|
||||
verify_commands = [str(cmd).strip() for cmd in (facts.get("verifyCommands") or []) if str(cmd).strip()]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
@@ -189,6 +190,24 @@ def test_nudge_explains_only_a_check_run_the_shell_could_not_attribute(tmp_path,
|
||||
assert "Your last check run (`pnpm test || true`) did not count" in hidden
|
||||
assert "did not count" not in scored
|
||||
|
||||
def test_nudge_drops_the_unscored_run_warning_once_hermes_verify_records(tmp_path, monkeypatch):
|
||||
"""``hermes verify`` records from its own process: its newer failure is the latest check."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
_node_project(tmp_path)
|
||||
changed = str(tmp_path / "src" / "app.ts")
|
||||
mark_workspace_edited(session_id="s1", cwd=tmp_path, paths=[changed])
|
||||
record_terminal_result(command="pnpm test || true", cwd=tmp_path, session_id="s1", exit_code=0)
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import sys; from agent.verification_evidence import record_verify_run; "
|
||||
"record_verify_run(root=sys.argv[1], session_id='s1', ok=False)", str(tmp_path)],
|
||||
cwd=Path(__file__).resolve().parents[2], check=True,
|
||||
)
|
||||
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
|
||||
|
||||
assert nudge is not None
|
||||
assert "hermes verify" in nudge and "did not count" not in nudge
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix C: documentation/prose edits carry no verifiable behavior and must never
|
||||
# trip the nudge, even on an unverified workspace.
|
||||
|
||||
Reference in new issue
Block a user