hermes-agent/hermes_state_errors.py

"""Exception types and error-classification predicates for the state store.
Shared by hermes_state and its mixins; string predicates match wrapped RPC
strings as well as live sqlite3 exceptions."""

import errno
import re
import sqlite3

# Malformed schema: ``sqlite_master`` itself is inconsistent (typically a DUPLICATE
# ``CREATE VIRTUAL TABLE messages_fts`` row). SQLite parses the whole schema while
# preparing the FIRST statement, so EVERY statement raises (even ``PRAGMA
# journal_mode`` during __init__); only ``PRAGMA writable_schema=ON`` +
# sqlite_master surgery still work. Canonical rows are intact; recovery rebuilds
# only the FTS layer.
_MALFORMED_SCHEMA_MARKERS = ("malformed database schema",)
_MALFORMED_DB_MARKERS = (*_MALFORMED_SCHEMA_MARKERS, "database disk image is malformed")


def is_malformed_db_error(exc: BaseException) -> bool:
    """Malformed-schema OR generic corrupt-image error. Diagnostics / offline
    recovery only — runtime repair must use :func:`is_malformed_schema_error`."""
    return isinstance(exc, sqlite3.DatabaseError) and any(
        marker in str(exc).lower() for marker in _MALFORMED_DB_MARKERS
    )


# SQLITE_IOERR as a substring (wrapped strings still classify).
_DISK_IO_ERROR_MARKER = "disk i/o error"

# "Store BUSY, not gone" — HTTP callers map these to 503 instead of 500. Corruption
# is deliberately absent: a malformed store must surface, not be retried into a timeout.
_TRANSIENT_SQLITE_MARKERS = (
    _DISK_IO_ERROR_MARKER, "database is locked", "database table is locked", "busy",
)


# Lock contention by result code. SQLite keeps SQLITE_BUSY when FTS5's xConnect loses the race
# on its %_config read but replaces the text with "vtable constructor failed: messages_fts",
# so a phrase match read a busy store as a hard failure.
_SQLITE_LOCK_CODES = (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)


def _sqlite_primary_code(exc_or_str) -> int | None:
    """Primary result code (extended codes keep it in the low byte); None when unknown."""
    code = getattr(exc_or_str, "sqlite_errorcode", None)
    return code & 0xFF if isinstance(code, int) else None


def is_sqlite_lock_error(exc_or_str) -> bool:
    """SQLITE_BUSY / SQLITE_LOCKED: wait and retry, never treat as damage. A known result code
    decides; only without one (our own re-raised messages, RPC-wrapped strings) does the text."""
    code = _sqlite_primary_code(exc_or_str)
    if code is not None:
        return code in _SQLITE_LOCK_CODES
    text = str(exc_or_str).lower()
    return "locked" in text or "busy" in text


def _is_no_more_rows(exc: sqlite3.Error) -> bool:
    """Transient engine error on contended WAL appends (retries like locked/busy);
    message-scoped because some builds raise it as InterfaceError."""
    return "no more rows available" in str(exc).lower()


def is_transient_sqlite_error(exc: BaseException) -> bool:
    """"Busy right now", not "damaged": one predicate so retry and the HTTP
    503-vs-500 split cannot drift apart."""
    return isinstance(exc, sqlite3.OperationalError) and (
        is_sqlite_lock_error(exc) or any(marker in str(exc).lower() for marker in _TRANSIENT_SQLITE_MARKERS)
    )


def is_malformed_schema_error(exc: BaseException) -> bool:
    """Only SQLite's explicit malformed-schema text: a generic "disk image is
    malformed" may be any B-tree page, so runtime repair must fail closed on it."""
    return isinstance(exc, sqlite3.DatabaseError) and any(
        marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS
    )


# "Filesystem cannot accept another write" substrings (OSError, sqlite3, wrapped RPC strings).
_DISK_FULL_MARKERS = (
    "no space left on device", "not enough space", "database or disk is full",  # SQLITE_FULL
    "disk full", "full disk", "enospc",
)


def is_disk_full_error(exc: BaseException | str | None) -> bool:
    """Disk-full / ENOSPC: OSError(ENOSPC), SQLITE_FULL, or matching strings."""
    if exc is None:
        return False
    if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC:
        return True
    lowered = (exc if isinstance(exc, str) else str(exc)).lower()
    return any(marker in lowered for marker in _DISK_FULL_MARKERS)


# Every classify_persistence_error bucket; consumers enumerate this tuple.
PERSISTENCE_ERROR_CAUSES = (
    "locked", "compression", "compression_closed", "turn_lease", "corrupt", "fts_index",
    "replaced", "deleted_wal", "disk", "session_row_missing", "unknown",
)


# "Database FILE structurally damaged" substrings. "database disk image is
# malformed" contains "disk", so this check MUST run before the disk bucket in
# classify_persistence_error or B-tree corruption reads as "free some disk space".
# Kept as plain substrings so sqlite3.DatabaseError, wrapped RPC strings, and logged message text all match
# the same helper. See #77386.
_DB_CORRUPTION_MARKERS = (
    "malformed", "file is not a database", "not a database", "database corruption",
)

# The module constant exists on Python 3.11+; the numeric value is stable across SQLite releases.
SQLITE_CORRUPT_VTAB = getattr(sqlite3, "SQLITE_CORRUPT_VTAB", 267)

# Every FTS object hangs off this prefix: the virtual tables and their _data/_idx/_content/
# _docsize/_config shadow b-trees. FTS5 names the table in its own corruption reports.
_FTS_OBJECT_RE = re.compile(r"\bmessages_fts\w*")


def is_fts_scoped_corruption_error(exc_or_str) -> bool:
    """Corruption SQLite itself attributes to the FTS index layer: the ONE provenance rule
    shared by the write-repair gate (``SessionDB._is_fts_write_corruption_error``), the
    gateway transcript retry and :func:`classify_persistence_error` (#96038, #97794).

    A known result code outranks prose: ``SQLITE_CORRUPT_VTAB`` is FTS-scoped even with
    the generic malformed-image text older SQLite builds emit, while bare ``SQLITE_CORRUPT``
    / ``SQLITE_NOTADB`` carry no object scope and any other known code contradicts
    FTS-looking prose, so both fail closed. Only without a code (Python < 3.11, RPC-wrapped
    strings) does the text decide, and then only an ``fts5:`` corruption report or a
    corruption marker that names a ``messages_fts*`` object counts.
    """
    if exc_or_str is None:
        return False
    code = getattr(exc_or_str, "sqlite_errorcode", None)
    if code is not None:
        return code == SQLITE_CORRUPT_VTAB
    text = (exc_or_str if isinstance(exc_or_str, str) else str(exc_or_str)).lower()
    if not _FTS_OBJECT_RE.search(text):
        return False
    if text.startswith("fts5:") and "corrupt" in text:
        return True
    return any(marker in text for marker in _DB_CORRUPTION_MARKERS)


class CompressionSessionClosedError(RuntimeError):
    """A durable write targeted a parent already closed by compression."""

    def __init__(self, session_id: str):
        self.session_id = session_id
        super().__init__(
            f"Session {session_id!r} is closed by compression; "
            "adopt its live continuation before appending messages"
        )


class CompressionSessionBusyError(RuntimeError):
    """A non-owner tried to write while compression owns the session."""


class SessionCompressionInProgressError(CompressionSessionBusyError):
    """A concurrent writer collided with a *live* compression lock — transient
    (the compressor publishes in seconds; ``_execute_write`` waits), unlike the
    parent class's other case (a compressor whose own lease is gone: permanent,
    fail fast). Subclassing keeps every existing handler working."""


class PruneRowUnresolvedError(RuntimeError):
    """An in-place prune rewrite could not name the exact live row of a changed message. Raised before
    any write, so the caller commits through ``archive_and_compact`` instead."""


class PruneRowStaleError(PruneRowUnresolvedError):
    """The named live row no longer holds the body the pruner is pruning: another writer already
    rewrote this generation. Unlike plain ambiguity this must NOT fall back to the full writer —
    that writer archives rows it never compared and would republish the stale held text over the
    newer one. Subclassing keeps callers that only know the parent failing closed-enough (they stop
    the in-place path); the prune caller tells the two apart and skips the pass entirely."""


class HeldRowRewrittenError(RuntimeError):
    """A full-writer commit found one of the held rows it was about to archive already rewritten in
    place by another writer. Raised INSIDE the commit's write transaction, after the caller's
    precheck but before any row is archived: a write that lands in that window is exactly the lost
    race the precheck cannot see (#124102). Callers treat it like ``StaleHeldHistory`` — nothing
    written, the pass is a no-op and the newer generation keeps the session."""


class SessionTurnLeaseLostError(RuntimeError):
    """A transcript write presented a turn-lease holder that no longer owns it.
    Fail-fast fencing (no ``_execute_write`` retry): a later writer may already
    be persisting a newer turn, and landing this one would interleave a stale reply."""


class SessionActiveWriteGuardError(RuntimeError):
    """Raised when an active turn lease or compression lock rejects session deletion."""


class StateDbReplacedError(RuntimeError):
    """The state.db path no longer names the file this SessionDB opened
    (out-of-band cp/mv/restore). In-place FTS repair and fail-open trigger
    dropping cannot fix a generation mismatch; they amplify it."""


class DeletedWalGenerationError(StateDbReplacedError):
    """A live process holds a deleted state.db-wal / -shm generation. Opening or
    writing through this handle would mint a second WAL inode (split-brain ->
    intermittent SQLITE_CORRUPT / IOERR). Stop the writers; never unlink the WAL
    yourself. Subclasses StateDbReplacedError so every consumer that diverts
    transcripts on a replaced store handles this identically."""


# SQLite header application_id (offset 68). Distinct from inode: ``cp`` onto the
# same path keeps st_ino and truncates+rewrites.
_STATE_DB_APPLICATION_ID_OFFSET = 68
_STATE_DB_GENERATION_KEY = "db_file_generation"
_STATE_DB_REPLACED_MSG = (
    "FATAL: state.db was replaced underneath the gateway; refusing further "
    "writes to this file. Divert transcripts to sessions/<id>.jsonl (and the "
    "gateway pending_messages spool) and restore or reopen after operator intervention."
)
STORAGE_RECOVERY_DOCS_URL = "https://hermes-agent.nousresearch.com/docs/user-guide/session-storage-recovery"

# Two layers (#110054): the first sentence is for the person reading a chat bubble or a banner (what
# happened, nothing is lost, the one thing to do); the rest is the operator detail. The phrase
# "deleted state.db-wal or state.db-shm" is the classifier's RPC-wrapped fingerprint — keep it.
_DELETED_WAL_GENERATION_MSG = (
    "FATAL: session storage stopped writing because another Hermes process still holds a deleted "
    "state.db-wal or state.db-shm inode (an old copy of the write-ahead log). Nothing is lost: quit "
    "every Hermes process on this profile (Desktop app, gateway, dashboard, cron), run `hermes doctor` "
    "(it names the processes still holding the log), then start Hermes again. Do not delete the WAL "
    "yourself and do not run `hermes doctor --fix` while they are running. "
    f"Guide: {STORAGE_RECOVERY_DOCS_URL} "
    "Detail: the path names a different (or missing) generation than the one this process holds "
    "open; opening or writing through it would mint a second WAL (split-brain). "
    "database.journal_mode: delete is operator containment, not a new default."
)


class StateDbCorruptError(sqlite3.DatabaseError):
    """A live SessionDB observed structural (non-FTS, non-replaced) corruption and
    is quarantined: sticky for the handle's life — writes fail fast, no reopen,
    no close-time checkpoint (a handle that kept writing after the first error
    checkpointed 15 pages under wrong page numbers and turned a readable file
    into "file is not a database"; SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE on 3.12+
    also stops SQLite's own). Subclasses sqlite3.DatabaseError so every degrade
    path keeps working. Recovery boundary: restart on a repaired/restored file.

    Stopping the writes is what prevents that; skipping the explicit checkpoint is the second line of
    defence. SQLite still runs its own last-connection checkpoint inside ``close()`` (and deletes the
    ``-wal`` sidecar) unless ``SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE`` is set — Python exposes it via
    ``Connection.setconfig()`` on 3.12+, so quarantine disables the close-time checkpoint there and the WAL
    survives on disk for forensics; on 3.11 the internal checkpoint is unavoidable (post-quarantine it can
    only carry pre-corruption committed frames, since no further writes are accepted). See #90837.
    """


_STATE_DB_CORRUPT_MSG = (
    "FATAL: state.db reported structural corruption (database disk image is "
    "malformed outside the FTS shadow tables) on a live handle; refusing further "
    "writes, automatic reopen, and the close-time WAL checkpoint on this file. "
    "Stop the gateway, then run `hermes sessions recover --source <state.db> "
    "--inspect-only` or restore a snapshot. Unwritten transcripts are diverted to "
    "sessions/<id>.jsonl (and the gateway pending_messages spool)."
)


_PERSISTENCE_CAUSE_BY_TYPE = (
    (SessionTurnLeaseLostError, "turn_lease"),
    (CompressionSessionClosedError, "compression_closed"),
    (CompressionSessionBusyError, "compression"),
    # The WAL-generation error subclasses StateDbReplacedError so existing write diversion keeps
    # working; classify it first because its recovery artifact and operator action are different.
    (DeletedWalGenerationError, "deleted_wal"),
    (StateDbReplacedError, "replaced"),
    (StateDbCorruptError, "corrupt"),
)
_PERSISTENCE_CAUSE_BY_PHRASE = (
    (("turn lease",), "turn_lease"),
    (("closed by compression",), "compression_closed"),
    (("being compressed", "compression lease"), "compression"),
    # RPC-wrapped errors lose their exception type; retain the same sidecar/main-file split.
    (("deleted state.db-wal", "deleted state.db-shm"), "deleted_wal"),
    (("was replaced underneath",), "replaced"),
    (_DB_CORRUPTION_MARKERS, "corrupt"),
    (("locked", "busy"), "locked"),
    # A flush rejected by the session-row FK: the row was removed under a live agent (#123583).
    (("foreign key constraint failed",), "session_row_missing"),
)


def classify_persistence_error(exc_or_str) -> str:
    """Coarse cause bucket (PERSISTENCE_ERROR_CAUSES) so the user's guidance
    matches: "locked" = busy, retry; "disk" = full/read-only/permissions;
    "compression" = a live lease refused the write; "compression_closed" = adopt
    the rotated session id; "turn_lease" = fencing, not storage; "corrupt" =
    file damage (repair path, not disk space); "fts_index" = SQLite scoped the
    corruption to the FTS index (the transcript store is not damaged); "replaced" =
    main-file replacement; "deleted_wal" = a retired sidecar generation requiring
    capture inspection; "session_row_missing" = the session row was deleted under a
    live agent (FK rejection; the flush recreates it)."""
    if exc_or_str is None:
        return "unknown"
    # Lease refusals contain neither "locked" nor "busy": match by type first,
    # then by phrase for strings that survived RPC wrapping. Order matters:
    # StateDbReplacedError covers DeletedWalGenerationError; corruption comes
    # BEFORE the lock/disk buckets ("disk image is malformed" contains "disk").
    for exc_type, cause in _PERSISTENCE_CAUSE_BY_TYPE:
        if isinstance(exc_or_str, exc_type):
            return cause
    # Provenance before prose: an FTS-scoped result code (or, without one, an fts5 report
    # naming messages_fts*) is index damage, never whole-file corruption (#97794).
    if is_fts_scoped_corruption_error(exc_or_str):
        return "fts_index"
    if _sqlite_primary_code(exc_or_str) in _SQLITE_LOCK_CODES:
        return "locked"
    if getattr(exc_or_str, "sqlite_errorcode", None) == sqlite3.SQLITE_CONSTRAINT_FOREIGNKEY:
        return "session_row_missing"
    text = str(exc_or_str).lower()
    for markers, cause in _PERSISTENCE_CAUSE_BY_PHRASE:
        if any(marker in text for marker in markers):
            return cause
    if is_disk_full_error(exc_or_str) or any(m in text for m in ("disk", "readonly", "read-only")):
        return "disk"
    return "unknown"