fix: SQLite WAL file can stay inflated on a running gateway until restart (#112951)

* fix: SQLite WAL file can stay inflated on a running gateway until restart

Since #82366 switched the periodic 30-minute checkpoint to PASSIVE (to keep
WAL maintenance off the event loop), no checkpoint on a running process
truncates the WAL *file* any more -- only close() does, i.e. a restart.
wal_autocheckpoint recycles WAL space in place but never shrinks the file,
and is itself a PASSIVE checkpoint a reader can transiently block. So when a
reader briefly pins frames (e.g. a memory reindex, a backup, a slow query),
the WAL grows past the autocheckpoint size and then stays parked at that
high-water mark for the whole life of the process. Observed in production: a
1.6 GB agent DB left a 1.6 GB -wal that only manual TRUNCATE checkpoints
could reclaim. This affects every SQLite-backed store (task registry, plugin
state, proxy capture, memory host, ...), not just memory.

Set PRAGMA journal_size_limit (default 64 MiB, overridable via
journalSizeLimitBytes) right after wal_autocheckpoint so any completing
checkpoint -- including the PASSIVE periodic/auto ones #82366 now relies on
-- truncates the WAL file back to the ceiling. This restores the bounded
on-disk WAL that TRUNCATE used to give, without reintroducing the blocking
checkpoint #82366 removed: journal_size_limit only changes how far a
completing checkpoint truncates, never checkpoint timing. The 64 MiB ceiling
sits ~16x above the autocheckpoint steady state (~4 MB at 1000 pages), so it
is inert in normal operation and engages only on pathological growth.

Related: #82366, #81715

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix: tighten SQLite WAL ceiling proof

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

docs/automation/tasks.md

@@ -319,7 +319,7 @@ Task records and delivery state persist in the shared OpenClaw SQLite state data
 
 Set `OPENCLAW_STATE_DIR` to move the whole state root (default `~/.openclaw`) elsewhere; the shared database path moves with it.
 
-The registry loads into memory on first use and persists every write back to SQLite, so records survive gateway restarts. WAL growth stays bounded through SQLite's default autocheckpoint threshold plus periodic `PASSIVE` checkpoints; shutdown and explicit maintenance checkpoints use `TRUNCATE` so normal closes reclaim WAL space without making the background sweeper wait on active readers.
+The registry loads into memory on first use and persists every write back to SQLite, so records survive gateway restarts. WAL growth stays bounded through SQLite's default autocheckpoint threshold plus periodic `PASSIVE` checkpoints. After a checkpoint completes, the next commit resets the WAL and applies a 64 MiB `journal_size_limit` ceiling, so a reader cannot leave the file parked at a pathological high-water mark until restart. Shutdown and explicit maintenance checkpoints use `TRUNCATE` so normal closes reclaim WAL space without making the background sweeper wait on active readers.
 
 Legacy sidecar stores from older installs (`tasks/runs.sqlite`, `flows/registry.sqlite`) are imported into the shared database by `openclaw doctor`.
 

src/infra/sqlite-pragma.test-support.ts

@@ -6,6 +6,7 @@ type SqliteNumberPragma =
   | "auto_vacuum"
   | "busy_timeout"
   | "foreign_keys"
+  | "journal_size_limit"
   | "schema_version"
   | "synchronous"
   | "user_version"

src/infra/sqlite-wal.test.ts

@@ -201,6 +201,51 @@ describe("sqlite WAL maintenance", () => {
     }
   });
 
+  it("reclaims an inflated WAL on the first commit after a completed checkpoint", () => {
+    const sqlite = requireNodeSqlite();
+    const dir = tempDirs.make("openclaw-sqlite-wal-size-");
+    const dbPath = path.join(dir, "openclaw.sqlite");
+    const walPath = `${dbPath}-wal`;
+    const db = new sqlite.DatabaseSync(dbPath);
+    let maintenance: ReturnType<typeof configureSqliteWalMaintenance> | undefined;
+    try {
+      maintenance = configureSqliteWalMaintenance(db, {
+        autoCheckpointPages: 0,
+        checkpointIntervalMs: 0,
+        databaseLabel: "wal-size-default",
+        databasePath: dbPath,
+      });
+      db.exec("CREATE TABLE payload (id INTEGER PRIMARY KEY, value TEXT NOT NULL);");
+      db.prepare("INSERT INTO payload (value) VALUES (?)").run("before-checkpoint");
+
+      const checkpoint = db.prepare("PRAGMA wal_checkpoint(PASSIVE);").get() as {
+        busy: number;
+        checkpointed: number;
+        log: number;
+      };
+      expect(checkpoint.busy).toBe(0);
+      expect(checkpoint.checkpointed).toBe(checkpoint.log);
+
+      const sizeLimit = Number(
+        (
+          db.prepare("PRAGMA journal_size_limit;").get() as {
+            journal_size_limit: number | bigint;
+          }
+        ).journal_size_limit,
+      );
+      expect(sizeLimit).toBe(64 * 1024 * 1024);
+      // A sparse extension models a retained high-water WAL without writing a 65 MiB fixture.
+      fs.truncateSync(walPath, sizeLimit + 1024 * 1024);
+
+      db.prepare("INSERT INTO payload (value) VALUES (?)").run("after-checkpoint");
+
+      expect(fs.statSync(walPath).size).toBe(sizeLimit);
+    } finally {
+      maintenance?.close();
+      db.close();
+    }
+  });
+
   it("rejects a memory journal for a file-backed database", () => {
     const db = createMockDb();
     vi.mocked(db["prepare"]).mockImplementation(
@@ -503,19 +548,20 @@ describe("sqlite WAL maintenance", () => {
     vi.spyOn(process, "platform", "get").mockReturnValue("linux");
 
     const maintenance = configureSqliteWalMaintenance(db, { checkpointIntervalMs: 100 });
-    expect(db["exec"]).toHaveBeenCalledTimes(2);
+    // journal_mode=WAL, wal_autocheckpoint, journal_size_limit.
+    expect(db["exec"]).toHaveBeenCalledTimes(3);
 
     vi.advanceTimersByTime(100);
     expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(PASSIVE);");
-    expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA incremental_vacuum(512);");
-    expect(db["exec"]).toHaveBeenCalledTimes(3);
+    expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);");
+    expect(db["exec"]).toHaveBeenCalledTimes(4);
 
     expect(maintenance.close()).toBe(true);
     expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(TRUNCATE);");
-    expect(db["exec"]).toHaveBeenCalledTimes(3);
+    expect(db["exec"]).toHaveBeenCalledTimes(4);
 
     vi.advanceTimersByTime(200);
-    expect(db["exec"]).toHaveBeenCalledTimes(3);
+    expect(db["exec"]).toHaveBeenCalledTimes(4);
   });
 
   it("clamps oversized checkpoint intervals before arming timers", () => {
@@ -544,7 +590,7 @@ describe("sqlite WAL maintenance", () => {
 
     vi.advanceTimersByTime(100);
     expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(FULL);");
-    expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA incremental_vacuum(512);");
+    expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);");
 
     expect(maintenance.close()).toBe(true);
     expect(db["prepare"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(FULL);");

src/infra/sqlite-wal.ts

@@ -10,6 +10,10 @@ import { isSqliteLockError } from "./sqlite-transaction.js";
 // checkpoints so state databases do not accumulate unbounded WAL files.
 const DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000;
 const DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS = 30 * 60 * 1000;
+// SQLite applies this ceiling when a fully checkpointed WAL resets on the next
+// commit. Keep it well above the usual ~4 MiB autocheckpoint window so only
+// pathological high-water marks pay the truncation cost.
+const DEFAULT_SQLITE_WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024;
 // 512 pages (~2MB at 4KB pages) per periodic pass keeps page release strictly
 // bounded so maintenance can never behave like a blocking full VACUUM.
 const INCREMENTAL_VACUUM_MAX_PAGES_PER_PASS = 512;
@@ -467,6 +471,7 @@ export function configureSqliteWalMaintenance(
   }
   enableMacosCheckpointFullfsync(db);
   db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`);
+  db.exec(`PRAGMA journal_size_limit = ${DEFAULT_SQLITE_WAL_JOURNAL_SIZE_LIMIT_BYTES};`);
 
   const runCheckpoint = (mode: SqliteWalCheckpointMode): boolean => {
     try {

src/state/openclaw-agent-db.test.ts

@@ -2605,6 +2605,7 @@ describe("openclaw agent database", () => {
     expect(readSqliteNumberPragma(database.db, "auto_vacuum")).toBe(2);
     expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
     expect(readSqliteNumberPragma(database.db, "wal_autocheckpoint")).toBe(1000);
+    expect(readSqliteNumberPragma(database.db, "journal_size_limit")).toBe(64 * 1024 * 1024);
     const journalMode = database.db.prepare("PRAGMA journal_mode").get() as
       | { journal_mode?: string }
       | undefined;

src/state/openclaw-state-db.test.ts

@@ -2530,7 +2530,10 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're
     const rmSync = fs.rmSync.bind(fs);
     let failRemoval = true;
     vi.spyOn(fs, "rmSync").mockImplementation(((pathname, options) => {
-      if (pathname === privateDirectory && failRemoval) {
+      if (
+        fs.realpathSync.native(String(pathname)) === fs.realpathSync.native(privateDirectory) &&
+        failRemoval
+      ) {
         failRemoval = false;
         const error = new Error("busy");
         (error as NodeJS.ErrnoException).code = "EBUSY";
@@ -3951,6 +3954,7 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're
     expect(readSqliteNumberPragma(database.db, "auto_vacuum")).toBe(2);
     expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION);
     expect(readSqliteNumberPragma(database.db, "wal_autocheckpoint")).toBe(1000);
+    expect(readSqliteNumberPragma(database.db, "journal_size_limit")).toBe(64 * 1024 * 1024);
     const journalMode = database.db.prepare("PRAGMA journal_mode").get() as
       | { journal_mode?: string }
       | undefined;