diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 8707b9136048..a1e9c24ec1d8 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -250,7 +250,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/session-store-runtime` | Session workflow helpers (`getSessionEntry`, `listSessionEntries`, `patchSessionEntry`, `upsertSessionEntry`), legacy session store path/session-key helpers, updated-at reads, and deprecated whole-store mutation helpers | | `plugin-sdk/cron-store-runtime` | Cron store path/load/save helpers | | `plugin-sdk/state-paths` | State/OAuth dir path helpers | - | `plugin-sdk/plugin-state-runtime` | Plugin sidecar SQLite keyed-state types | + | `plugin-sdk/plugin-state-runtime` | Plugin sidecar SQLite keyed-state types plus centralized connection pragma and WAL maintenance setup for plugin-owned databases | | `plugin-sdk/routing` | Route/session-key/account binding helpers such as `resolveAgentRoute`, `buildAgentSessionKey`, and `resolveDefaultAgentBoundAccountId` | | `plugin-sdk/status-helpers` | Shared channel/account status summary helpers, runtime-state defaults, and issue metadata helpers | | `plugin-sdk/target-resolver-runtime` | Shared target resolver helpers | diff --git a/extensions/workboard/src/sqlite-store.ts b/extensions/workboard/src/sqlite-store.ts index cd1bca1b9554..d4617ef55633 100644 --- a/extensions/workboard/src/sqlite-store.ts +++ b/extensions/workboard/src/sqlite-store.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import { configureSqliteConnectionPragmas } from "openclaw/plugin-sdk/plugin-state-runtime"; import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; import type { PersistedWorkboardAttachment, @@ -361,15 +362,6 @@ function ensureWorkboardSchema(db: DatabaseSync): void { ).run(`schema-${SCHEMA_VERSION}`, Date.now()); } -function configureWorkboardDatabase(db: DatabaseSync): void { - db.exec(` - PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; - PRAGMA busy_timeout = ${WORKBOARD_SQLITE_BUSY_TIMEOUT_MS}; - PRAGMA foreign_keys = ON; - `); -} - function chmodIfExists(targetPath: string, mode: number): void { try { fs.chmodSync(targetPath, mode); @@ -385,19 +377,30 @@ function hardenWorkboardDatabaseFiles(dbPath: string): void { chmodIfExists(dbPath, WORKBOARD_SQLITE_FILE_MODE); chmodIfExists(`${dbPath}-wal`, WORKBOARD_SQLITE_FILE_MODE); chmodIfExists(`${dbPath}-shm`, WORKBOARD_SQLITE_FILE_MODE); + chmodIfExists(`${dbPath}-journal`, WORKBOARD_SQLITE_FILE_MODE); } -function createDatabase(dbPath: string): DatabaseSync { +function createDatabase(dbPath: string): { + db: DatabaseSync; + maintenance: ReturnType; +} { fs.mkdirSync(path.dirname(dbPath), { recursive: true, mode: WORKBOARD_SQLITE_DIR_MODE }); chmodIfExists(path.dirname(dbPath), WORKBOARD_SQLITE_DIR_MODE); if (!fs.existsSync(dbPath)) { fs.closeSync(fs.openSync(dbPath, "a", WORKBOARD_SQLITE_FILE_MODE)); } const db = new DatabaseSync(dbPath); - configureWorkboardDatabase(db); + const maintenance = configureSqliteConnectionPragmas(db, { + busyTimeoutMs: WORKBOARD_SQLITE_BUSY_TIMEOUT_MS, + checkpointIntervalMs: 0, + databaseLabel: "workboard database", + databasePath: dbPath, + foreignKeys: true, + synchronous: "NORMAL", + }); ensureWorkboardSchema(db); hardenWorkboardDatabaseFiles(dbPath); - return db; + return { db, maintenance }; } function childRows(db: DatabaseSync, table: string, cardId: string): Row[] { @@ -1401,12 +1404,17 @@ export function createWorkboardSqliteStores( env?: NodeJS.ProcessEnv; } = {}, ): WorkboardSqliteStores { - const db = createDatabase(options.dbPath ?? resolveWorkboardSqlitePath(options.env)); + const { db, maintenance } = createDatabase( + options.dbPath ?? resolveWorkboardSqlitePath(options.env), + ); return { cards: new WorkboardSqliteCardStore(db), boards: new WorkboardSqliteBoardStore(db), subscriptions: new WorkboardSqliteSubscriptionStore(db), attachments: new WorkboardSqliteAttachmentStore(db), - close: () => db.close(), + close: () => { + maintenance.close(); + db.close(); + }, }; } diff --git a/extensions/workboard/src/store.test.ts b/extensions/workboard/src/store.test.ts index 216e593631d9..e1967cf3ada6 100644 --- a/extensions/workboard/src/store.test.ts +++ b/extensions/workboard/src/store.test.ts @@ -36,6 +36,18 @@ function createMemoryStore(options?: { }; } +function statfsFixture(type: number): ReturnType { + return { + type, + bsize: 1024, + blocks: 1, + bfree: 1, + bavail: 1, + files: 0, + ffree: 0, + }; +} + describe("WorkboardStore", () => { it("persists boards, cards, subscriptions, and attachment blobs in sqlite", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-workboard-sqlite-")); @@ -90,7 +102,7 @@ describe("WorkboardStore", () => { if (process.platform !== "win32") { expect(fs.statSync(dir).mode & 0o777).toBe(0o700); expect(fs.statSync(dbPath).mode & 0o777).toBe(0o600); - for (const sidecarPath of [`${dbPath}-wal`, `${dbPath}-shm`]) { + for (const sidecarPath of [`${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-journal`]) { if (fs.existsSync(sidecarPath)) { expect(fs.statSync(sidecarPath).mode & 0o777).toBe(0o600); } @@ -144,6 +156,27 @@ describe("WorkboardStore", () => { } }); + it("uses rollback journaling on network-backed volumes", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-workboard-sqlite-network-")); + const dbPath = path.join(dir, "workboard.sqlite"); + const statfs = vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0xff534d42)); + try { + const stores = createWorkboardSqliteStores({ dbPath }); + stores.close(); + + const rawDb = new DatabaseSync(dbPath); + expect(rawDb.prepare("PRAGMA journal_mode").get()).toMatchObject({ + journal_mode: "delete", + }); + rawDb.close(); + expect(fs.existsSync(`${dbPath}-wal`)).toBe(false); + expect(fs.existsSync(`${dbPath}-shm`)).toBe(false); + } finally { + statfs.mockRestore(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("creates and lists cards by status order and position", async () => { const store = new WorkboardStore(createMemoryStore()); diff --git a/src/infra/sqlite-wal.test.ts b/src/infra/sqlite-wal.test.ts index c70f857f21a7..4e68db4c3dbb 100644 --- a/src/infra/sqlite-wal.test.ts +++ b/src/infra/sqlite-wal.test.ts @@ -73,7 +73,97 @@ describe("sqlite WAL maintenance", () => { } }); - it("refuses NFS-backed databases when SQLite keeps WAL active", () => { + it.each([ + ["SMB", 0x517b], + ["CIFS", 0xff534d42], + ["SMB2", 0xfe534d42], + ])("uses rollback journaling for databases on Linux %s volumes", (_label, fsType) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-network-")); + try { + const db = createMockDb(); + vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(fsType)); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath: path.join(tempDir, "openclaw.sqlite"), + }); + + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;"); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it.each([ + String.raw`\\server\share\openclaw.sqlite`, + String.raw`\\?\UNC\server\share\openclaw.sqlite`, + "//server/share/openclaw.sqlite", + "//?/UNC/server/share/openclaw.sqlite", + ])("uses rollback journaling for databases on Windows UNC paths: %s", (databasePath) => { + const db = createMockDb(); + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath, + }); + + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;"); + expect(db["exec"]).not.toHaveBeenCalled(); + }); + + it("uses rollback journaling for mapped Windows network drives", () => { + const db = createMockDb(); + const databasePath = String.raw`Z:\state\openclaw.sqlite`; + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const realpath = vi + .spyOn(fs.realpathSync, "native") + .mockReturnValue(String.raw`\\server\share\state\openclaw.sqlite`); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath, + }); + + expect(realpath).toHaveBeenCalledWith(databasePath); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;"); + expect(db["exec"]).not.toHaveBeenCalled(); + }); + + it("does not treat namespaced Windows local drives as UNC paths", () => { + const db = createMockDb(); + const databasePath = String.raw`\\?\C:\state\openclaw.sqlite`; + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const realpath = vi.spyOn(fs.realpathSync, "native").mockReturnValue(databasePath); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath, + }); + + expect(realpath).toHaveBeenCalledWith(databasePath); + expect(db["prepare"]).not.toHaveBeenCalled(); + expect(db["exec"]).toHaveBeenNthCalledWith(1, "PRAGMA journal_mode = WAL;"); + }); + + it("uses rollback journaling when Windows cannot classify an opened drive path", () => { + const db = createMockDb(); + const databasePath = String.raw`Z:\restricted\openclaw.sqlite`; + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + vi.spyOn(fs.realpathSync, "native").mockImplementation(() => { + throw new Error("access denied"); + }); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath, + }); + + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;"); + expect(db["exec"]).not.toHaveBeenCalled(); + }); + + it("refuses network-backed databases when SQLite keeps WAL active", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-")); try { const db = createMockDb(); @@ -137,6 +227,29 @@ describe("sqlite WAL maintenance", () => { } }); + it("uses macOS SMB mount filesystem names", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-smb-")); + try { + const db = createMockDb(); + vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0)); + vi.spyOn(fs, "readFileSync").mockImplementation(() => { + throw new Error("no proc mountinfo"); + }); + vi.spyOn(childProcess, "execFileSync").mockReturnValue( + Buffer.from(`//server/share on ${tempDir} (smbfs, nodev, nosuid)\n`), + ); + + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databasePath: path.join(tempDir, "openclaw.sqlite"), + }); + + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;"); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + it("parses Linux mount command filesystem names when proc mountinfo is unavailable", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-")); try { diff --git a/src/infra/sqlite-wal.ts b/src/infra/sqlite-wal.ts index 0d2662deae31..7c23ccbc0f02 100644 --- a/src/infra/sqlite-wal.ts +++ b/src/infra/sqlite-wal.ts @@ -15,7 +15,11 @@ export const DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS = 30 * 60 * 1000; */ export const DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS = DEFAULT_SQLITE_WAL_CHECKPOINT_INTERVAL_MS; const LINUX_NFS_SUPER_MAGIC = 0x6969; +const LINUX_SMB_SUPER_MAGIC = 0x517b; +const LINUX_CIFS_SUPER_MAGIC = 0xff534d42; +const LINUX_SMB2_SUPER_MAGIC = 0xfe534d42; const PROC_MOUNTINFO_PATH = "/proc/self/mountinfo"; +const NETWORK_FILESYSTEM_TYPES = new Set(["cifs", "smbfs", "smb2", "smb3"]); type IntervalHandle = ReturnType & { unref?: () => void; @@ -133,33 +137,66 @@ function isPathWithinMount(targetPath: string, mountPoint: string): boolean { ); } -function isNfsMountType(fsType: string): boolean { - return fsType.toLowerCase().startsWith("nfs"); +function isNetworkMountType(fsType: string): boolean { + const normalized = fsType.toLowerCase(); + return normalized.startsWith("nfs") || NETWORK_FILESYSTEM_TYPES.has(normalized); } -function isNfsMountEntryPath(targetPath: string): boolean { +function isNetworkMountEntryPath(targetPath: string): boolean { const mountEntry = readMountEntries() .filter((entry) => isPathWithinMount(targetPath, entry.mountPoint)) .toSorted((a, b) => b.mountPoint.length - a.mountPoint.length)[0]; - return mountEntry ? isNfsMountType(mountEntry.fsType) : false; + return mountEntry ? isNetworkMountType(mountEntry.fsType) : false; } -function isNfsBackedPath(targetPath: string): boolean { +function isWindowsUncPath(targetPath: string): boolean { + return ( + /^\\\\\?\\UNC\\[^\\]+\\[^\\]+/i.test(targetPath) || + /^\\\\(?![?.]\\)[^\\]+\\[^\\]+/.test(targetPath) + ); +} + +function isWindowsDrivePath(targetPath: string): boolean { + return /^[A-Za-z]:[\\/]/.test(targetPath) || /^\\\\\?\\[A-Za-z]:[\\/]/i.test(targetPath); +} + +function isNetworkBackedPath(targetPath: string): boolean { + if (process.platform === "win32") { + const normalizedTargetPath = path.win32.normalize(targetPath); + if (isWindowsUncPath(normalizedTargetPath)) { + return true; + } + if (isWindowsDrivePath(normalizedTargetPath)) { + try { + return isWindowsUncPath(path.win32.normalize(fs.realpathSync.native(targetPath))); + } catch { + // Windows can deny SMB path normalization when parent components are + // unreadable. Treat an unclassifiable opened database as network-backed. + return true; + } + } + } if (typeof fs.statfsSync !== "function") { - return isNfsMountEntryPath(targetPath); + return isNetworkMountEntryPath(targetPath); } const checkedPath = findExistingVolumePath(targetPath); if (!checkedPath) { return false; } try { - if (fs.statfsSync(checkedPath).type === LINUX_NFS_SUPER_MAGIC) { + const filesystemType = fs.statfsSync(checkedPath).type; + if ( + filesystemType === LINUX_NFS_SUPER_MAGIC || + filesystemType === LINUX_SMB_SUPER_MAGIC || + filesystemType === LINUX_CIFS_SUPER_MAGIC || + filesystemType === LINUX_SMB2_SUPER_MAGIC + ) { return true; } } catch { - return isNfsMountEntryPath(checkedPath); + return isNetworkMountEntryPath(checkedPath); } - return isNfsMountEntryPath(checkedPath); + return isNetworkMountEntryPath(checkedPath); } function readJournalModeResult(row: unknown): string | null { @@ -179,7 +216,7 @@ function requireRollbackJournalMode(db: DatabaseSync, options: SqliteWalMaintena const location = options.databasePath ? ` at ${options.databasePath}` : ""; const actual = journalMode ?? "unknown"; throw new Error( - `${label}${location} is on an NFS-backed volume but SQLite kept journal_mode=${actual}; refusing to continue with WAL on NFS.`, + `${label}${location} is on a network-backed volume but SQLite kept journal_mode=${actual}; refusing to continue with WAL on network storage.`, ); } } @@ -200,7 +237,7 @@ export function configureSqliteWalMaintenance( const timerIntervalMs = Math.min(checkpointIntervalMs, MAX_TIMER_TIMEOUT_MS); const checkpointMode = options.checkpointMode ?? "TRUNCATE"; const periodicCheckpointMode = options.checkpointMode ?? "PASSIVE"; - if (options.databasePath && isNfsBackedPath(options.databasePath)) { + if (options.databasePath && isNetworkBackedPath(options.databasePath)) { requireRollbackJournalMode(db, options); return { checkpoint: () => true, diff --git a/src/plugin-sdk/plugin-state-runtime.ts b/src/plugin-sdk/plugin-state-runtime.ts index 26fd973739f7..82a4feffde6a 100644 --- a/src/plugin-sdk/plugin-state-runtime.ts +++ b/src/plugin-sdk/plugin-state-runtime.ts @@ -1,6 +1,7 @@ /** * Runtime SDK type surface for plugin-scoped keyed state stores. */ +export { configureSqliteConnectionPragmas } from "../infra/sqlite-wal.js"; export type { OpenKeyedStoreOptions, PluginStateEntry,