fix(state): harden sqlite path caching

Resolve explicit relative SQLite DB paths before caching handles and centralize durable SQLite connection pragmas so busy_timeout is applied before WAL/NFS negotiation.
This commit is contained in:
Vincent Koc
2026-06-15 01:04:35 +08:00
committed by GitHub
parent 7e12a3326d
commit b470316fc0
13 changed files with 270 additions and 30 deletions

View File

@@ -127,11 +127,10 @@ export function cleanupAgedMemoryReindexTempFiles(dbPath: string, nowMs = Date.n
function openConfiguredMemoryDatabaseAtPath(dbPath: string, allowExtension: boolean): DatabaseSync {
const { DatabaseSync } = requireNodeSqlite();
const db = new DatabaseSync(dbPath, { allowExtension });
configureMemorySqliteWalMaintenance(db, { databasePath: dbPath });
// busy_timeout is per-connection and resets to 0 on restart.
// Set it on every open so concurrent processes retry instead of
// failing immediately with SQLITE_BUSY.
db.exec("PRAGMA busy_timeout = 5000");
configureMemorySqliteWalMaintenance(db, {
busyTimeoutMs: 5000,
databasePath: dbPath,
});
return db;
}

View File

@@ -6,6 +6,7 @@ export {
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
applyWindowsSpawnProgramPolicy,
configureSqliteConnectionPragmas,
configureSqliteWalMaintenance,
root,
createSubsystemLogger,
@@ -31,6 +32,7 @@ export type {
ProcessWarning,
ResolveWindowsSpawnProgramCandidateParams,
ResolveWindowsSpawnProgramParams,
SqliteConnectionPragmaOptions,
SqliteWalMaintenance,
SqliteWalMaintenanceOptions,
WindowsSpawnCandidateResolution,

View File

@@ -82,9 +82,11 @@ export {
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
configureSqliteConnectionPragmas,
configureSqliteWalMaintenance,
} from "../../../../src/infra/sqlite-wal.js";
export type {
SqliteConnectionPragmaOptions,
SqliteWalMaintenance,
SqliteWalMaintenanceOptions,
} from "../../../../src/infra/sqlite-wal.js";

View File

@@ -4,6 +4,11 @@ export {
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS,
DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS,
configureSqliteConnectionPragmas,
configureSqliteWalMaintenance,
} from "./openclaw-runtime-io.js";
export type { SqliteWalMaintenance, SqliteWalMaintenanceOptions } from "./openclaw-runtime-io.js";
export type {
SqliteConnectionPragmaOptions,
SqliteWalMaintenance,
SqliteWalMaintenanceOptions,
} from "./openclaw-runtime-io.js";

View File

@@ -3,7 +3,9 @@ import { createRequire } from "node:module";
import type { DatabaseSync } from "node:sqlite";
import { formatErrorMessage } from "./error-utils.js";
import {
configureSqliteConnectionPragmas,
configureSqliteWalMaintenance,
type SqliteConnectionPragmaOptions,
type SqliteWalMaintenance,
type SqliteWalMaintenanceOptions,
} from "./sqlite-wal.js";
@@ -29,13 +31,16 @@ export function requireNodeSqlite(): typeof import("node:sqlite") {
export function configureMemorySqliteWalMaintenance(
db: DatabaseSync,
options?: SqliteWalMaintenanceOptions,
options?: SqliteWalMaintenanceOptions & Pick<SqliteConnectionPragmaOptions, "busyTimeoutMs">,
): SqliteWalMaintenance {
const existing = sqliteWalMaintenanceByDb.get(db);
if (existing) {
return existing;
}
const maintenance = configureSqliteWalMaintenance(db, options);
const maintenance =
options?.busyTimeoutMs === undefined
? configureSqliteWalMaintenance(db, options)
: configureSqliteConnectionPragmas(db, options);
sqliteWalMaintenanceByDb.set(db, maintenance);
return maintenance;
}

View File

@@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js";
import {
DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES,
configureSqliteConnectionPragmas,
configureSqliteWalMaintenance,
} from "./sqlite-wal.js";
@@ -225,4 +226,45 @@ describe("sqlite WAL maintenance", () => {
expect(maintenance.checkpoint()).toBe(false);
expect(onCheckpointError).toHaveBeenCalledWith(error);
});
it("configures connection pragmas before WAL maintenance", () => {
const db = createMockDb();
configureSqliteConnectionPragmas(db, {
busyTimeoutMs: 30_000,
checkpointIntervalMs: 0,
foreignKeys: true,
synchronous: "NORMAL",
});
expect(db["exec"]).toHaveBeenNthCalledWith(1, "PRAGMA busy_timeout = 30000;");
expect(db["exec"]).toHaveBeenNthCalledWith(2, "PRAGMA journal_mode = WAL;");
expect(db["exec"]).toHaveBeenNthCalledWith(
3,
`PRAGMA wal_autocheckpoint = ${DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES};`,
);
expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA synchronous = NORMAL;");
expect(db["exec"]).toHaveBeenNthCalledWith(5, "PRAGMA foreign_keys = ON;");
});
it("sets busy timeout before rollback journaling on NFS-backed volumes", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
try {
const db = createMockDb();
vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0x6969));
configureSqliteConnectionPragmas(db, {
busyTimeoutMs: 5000,
checkpointIntervalMs: 0,
databasePath: path.join(tempDir, "openclaw.sqlite"),
synchronous: "NORMAL",
});
expect(db["exec"]).toHaveBeenNthCalledWith(1, "PRAGMA busy_timeout = 5000;");
expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;");
expect(db["exec"]).toHaveBeenNthCalledWith(2, "PRAGMA synchronous = NORMAL;");
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
});

View File

@@ -38,6 +38,12 @@ export type SqliteWalMaintenanceOptions = {
onCheckpointError?: (error: unknown) => void;
};
export type SqliteConnectionPragmaOptions = SqliteWalMaintenanceOptions & {
busyTimeoutMs?: number;
foreignKeys?: boolean;
synchronous?: "NORMAL";
};
function normalizeNonNegativeInteger(value: number, label: string): number {
if (!Number.isInteger(value) || value < 0) {
throw new Error(`${label} must be a non-negative integer`);
@@ -236,3 +242,24 @@ export function configureSqliteWalMaintenance(
},
};
}
/** Configure per-connection SQLite pragmas in the safe lock-retry/WAL order. */
export function configureSqliteConnectionPragmas(
db: DatabaseSync,
options: SqliteConnectionPragmaOptions = {},
): SqliteWalMaintenance {
const { busyTimeoutMs, foreignKeys, synchronous, ...walOptions } = options;
if (busyTimeoutMs !== undefined) {
db.exec(
`PRAGMA busy_timeout = ${normalizeNonNegativeInteger(busyTimeoutMs, "busyTimeoutMs")};`,
);
}
const maintenance = configureSqliteWalMaintenance(db, walOptions);
if (synchronous) {
db.exec(`PRAGMA synchronous = ${synchronous};`);
}
if (foreignKeys) {
db.exec("PRAGMA foreign_keys = ON;");
}
return maintenance;
}

View File

@@ -5,7 +5,10 @@ import type { DatabaseSync } from "node:sqlite";
import { normalizeNullableString as normalizeObservedValue } from "@openclaw/normalization-core/string-coerce";
import { normalizeUniqueStringEntries } from "@openclaw/normalization-core/string-normalization";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js";
import {
configureSqliteConnectionPragmas,
type SqliteWalMaintenance,
} from "../infra/sqlite-wal.js";
import { readCaptureBlobText, writeCaptureBlob } from "./blob-store.js";
import type {
CaptureBlobRecord,
@@ -33,11 +36,11 @@ function openDatabase(dbPath: string): OpenedDatabase {
ensureParentDir(dbPath);
const { DatabaseSync } = requireNodeSqlite();
const db = new DatabaseSync(dbPath);
const walMaintenance = configureSqliteWalMaintenance(db, {
const walMaintenance = configureSqliteConnectionPragmas(db, {
busyTimeoutMs: 5000,
databaseLabel: "debug-proxy-capture",
databasePath: dbPath,
});
db.exec("PRAGMA busy_timeout = 5000");
db.exec(`
CREATE TABLE IF NOT EXISTS capture_sessions (
id TEXT PRIMARY KEY,

View File

@@ -19,15 +19,15 @@ export type OpenClawAgentSqlitePathOptions = {
/** Resolve the SQLite file for one normalized agent id. */
export function resolveOpenClawAgentSqlitePath(options: OpenClawAgentSqlitePathOptions): string {
const agentId = normalizeAgentId(options.agentId);
return (
return path.resolve(
options.path ??
path.join(
path.dirname(resolveOpenClawStateSqliteDir(options.env ?? process.env)),
"agents",
agentId,
"agent",
"openclaw-agent.sqlite",
)
path.join(
path.dirname(resolveOpenClawStateSqliteDir(options.env ?? process.env)),
"agents",
agentId,
"agent",
"openclaw-agent.sqlite",
),
);
}

View File

@@ -1,4 +1,5 @@
// OpenClaw agent database tests cover agent-scoped DB storage and migrations.
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -154,6 +155,82 @@ describe("openclaw agent database", () => {
).toThrow(/run openclaw doctor --fix/);
});
it("keys explicit relative paths by resolved database pathname", () => {
const agentModuleUrl = new URL("./openclaw-agent-db.ts", import.meta.url).href;
const stateModuleUrl = new URL("./openclaw-state-db.ts", import.meta.url).href;
const output = execFileSync(
process.execPath,
[
"--import",
"tsx",
"--input-type=module",
"-e",
`
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
closeOpenClawAgentDatabasesForTest,
listOpenClawRegisteredAgentDatabases,
openOpenClawAgentDatabase,
} from ${JSON.stringify(agentModuleUrl)};
import { closeOpenClawStateDatabaseForTest } from ${JSON.stringify(stateModuleUrl)};
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-db-state-"));
const env = { OPENCLAW_STATE_DIR: stateDir };
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-db-relative-"));
const firstDir = path.join(root, "first");
const secondDir = path.join(root, "second");
fs.mkdirSync(firstDir);
fs.mkdirSync(secondDir);
const previousCwd = process.cwd();
try {
process.chdir(firstDir);
const first = openOpenClawAgentDatabase({
agentId: "worker-1",
env,
path: "agent.sqlite",
});
process.chdir(secondDir);
const second = openOpenClawAgentDatabase({
agentId: "worker-1",
env,
path: "agent.sqlite",
});
console.log(JSON.stringify({
sameHandle: first === second,
firstFileExists: fs.existsSync(path.join(firstDir, "agent.sqlite")),
secondFileExists: fs.existsSync(path.join(secondDir, "agent.sqlite")),
registeredPaths: listOpenClawRegisteredAgentDatabases({ env })
.filter((entry) => entry.agentId === "worker-1")
.map((entry) => entry.path),
expectedPaths: [first.path, second.path].toSorted(),
}));
} finally {
process.chdir(previousCwd);
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
}
`,
],
{ encoding: "utf8" },
);
const result = JSON.parse(output) as {
expectedPaths: string[];
firstFileExists: boolean;
registeredPaths: string[];
sameHandle: boolean;
secondFileExists: boolean;
};
expect(result.sameHandle).toBe(false);
expect(result.firstFileExists).toBe(true);
expect(result.secondFileExists).toBe(true);
expect(result.registeredPaths).toEqual(result.expectedPaths);
});
it("rejects sharing one explicit database path across agent ids", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };

View File

@@ -9,7 +9,10 @@ import {
} from "../infra/kysely-sync.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js";
import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js";
import {
configureSqliteConnectionPragmas,
type SqliteWalMaintenance,
} from "../infra/sqlite-wal.js";
import { normalizeAgentId } from "../routing/session-key.js";
import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js";
import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
@@ -265,13 +268,13 @@ export function openOpenClawAgentDatabase(
ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions);
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
const walMaintenance = configureSqliteWalMaintenance(db, {
const walMaintenance = configureSqliteConnectionPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: `openclaw-agent:${agentId}`,
databasePath: pathname,
foreignKeys: true,
synchronous: "NORMAL",
});
db.exec("PRAGMA synchronous = NORMAL;");
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
db.exec("PRAGMA foreign_keys = ON;");
try {
ensureAgentSchema(db, agentId, pathname);
} catch (err) {

View File

@@ -1,4 +1,5 @@
// OpenClaw state database tests cover state DB migrations and persistence.
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
@@ -406,6 +407,77 @@ describe("openclaw state database", () => {
expect(readSqliteNumberPragma(first.db, "user_version")).toBe(1);
});
it("keys explicit relative paths by resolved database pathname", () => {
const moduleUrl = new URL("./openclaw-state-db.ts", import.meta.url).href;
const output = execFileSync(
process.execPath,
[
"--import",
"tsx",
"--input-type=module",
"-e",
`
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from ${JSON.stringify(moduleUrl)};
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-state-db-relative-"));
const firstDir = path.join(root, "first");
const secondDir = path.join(root, "second");
fs.mkdirSync(firstDir);
fs.mkdirSync(secondDir);
const previousCwd = process.cwd();
try {
process.chdir(firstDir);
const firstPath = path.resolve("state.sqlite");
const first = openOpenClawStateDatabase({ path: "state.sqlite" });
first.db
.prepare("INSERT INTO diagnostic_events (scope, event_key, payload_json, created_at) VALUES (?, ?, ?, ?)")
.run("relative-path", "first", "{}", 1);
process.chdir(secondDir);
const secondPath = path.resolve("state.sqlite");
const second = openOpenClawStateDatabase({ path: "state.sqlite" });
second.db
.prepare("INSERT INTO diagnostic_events (scope, event_key, payload_json, created_at) VALUES (?, ?, ?, ?)")
.run("relative-path", "second", "{}", 2);
console.log(JSON.stringify({
sameHandle: first === second,
firstPath,
secondPath,
firstFileExists: fs.existsSync(path.join(firstDir, "state.sqlite")),
secondFileExists: fs.existsSync(path.join(secondDir, "state.sqlite")),
firstRows: first.db.prepare("SELECT event_key FROM diagnostic_events WHERE scope = ?").all("relative-path"),
secondRows: second.db.prepare("SELECT event_key FROM diagnostic_events WHERE scope = ?").all("relative-path"),
}));
} finally {
process.chdir(previousCwd);
closeOpenClawStateDatabaseForTest();
}
`,
],
{ encoding: "utf8" },
);
const result = JSON.parse(output) as {
firstFileExists: boolean;
firstRows: Array<{ event_key: string }>;
sameHandle: boolean;
secondFileExists: boolean;
secondRows: Array<{ event_key: string }>;
};
expect(result.sameHandle).toBe(false);
expect(result.firstFileExists).toBe(true);
expect(result.secondFileExists).toBe(true);
expect(result.firstRows).toEqual([{ event_key: "first" }]);
expect(result.secondRows).toEqual([{ event_key: "second" }]);
});
it("uses savepoints for nested write transaction rollback", () => {
const stateDir = createTempStateDir();
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };

View File

@@ -10,7 +10,10 @@ import {
} from "../infra/kysely-sync.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js";
import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js";
import {
configureSqliteConnectionPragmas,
type SqliteWalMaintenance,
} from "../infra/sqlite-wal.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
@@ -892,7 +895,7 @@ function ensureSchema(db: DatabaseSync, pathname: string): void {
}
function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string {
return options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env);
return path.resolve(options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env));
}
/** Open or return a cached shared state database after schema and migration checks. */
@@ -915,13 +918,13 @@ export function openOpenClawStateDatabase(
ensureOpenClawStatePermissions(pathname, env);
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
const walMaintenance = configureSqliteWalMaintenance(db, {
const walMaintenance = configureSqliteConnectionPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: "openclaw-state",
databasePath: pathname,
foreignKeys: true,
synchronous: "NORMAL",
});
db.exec("PRAGMA synchronous = NORMAL;");
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
db.exec("PRAGMA foreign_keys = ON;");
try {
ensureSchema(db, pathname);
} catch (err) {