From 88bc08c124a592e64451cdd866d19e01314d7ade Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 17 Jun 2026 06:35:46 +0200 Subject: [PATCH] refactor(memory): canonicalize agent database tables --- .../memory-core/src/memory/index.test.ts | 39 +++++--- .../memory-core/src/memory/manager-db.test.ts | 73 +++++++++++--- .../memory-core/src/memory/manager-db.ts | 36 +++---- .../memory/manager-embedding-cache.test.ts | 2 - .../src/memory/manager-embedding-cache.ts | 4 +- .../src/memory/manager-embedding-ops.ts | 23 +++-- .../src/memory/manager-fts-state.test.ts | 24 +++-- .../src/memory/manager-fts-state.ts | 2 +- .../src/memory/manager-search.test.ts | 74 ++++++-------- .../memory-core/src/memory/manager-search.ts | 10 +- .../src/memory/manager-source-state.ts | 4 +- .../src/memory/manager-status-state.ts | 4 +- .../src/memory/manager-sync-ops.ts | 35 +++---- .../src/memory/manager-vector-warning.test.ts | 4 +- .../src/memory/manager-vector-warning.ts | 2 +- .../src/memory/manager-vector-write.ts | 2 +- .../memory/manager.fts-only-reindex.test.ts | 2 +- .../memory/manager.reindex-recovery.test.ts | 10 +- ...manager.self-heal-missing-identity.test.ts | 10 +- extensions/memory-core/src/memory/manager.ts | 11 ++- .../src/memory/manager.vector-dedupe.test.ts | 10 +- .../memory-host-sdk/src/engine-storage.ts | 11 ++- .../memory-host-sdk/src/host/memory-schema.ts | 97 ++++++++----------- src/commands/status.scan.shared.ts | 26 +++-- src/memory-host-sdk/engine-storage.ts | 3 + .../memory-core-host-engine-storage.ts | 7 ++ src/state/openclaw-agent-db.generated.d.ts | 46 +++++++++ src/state/openclaw-agent-schema.generated.ts | 91 ++++++++++++++++- src/state/openclaw-agent-schema.sql | 89 +++++++++++++++++ 29 files changed, 522 insertions(+), 229 deletions(-) diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index 8746d18e86fa..c332f742fff3 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -341,7 +341,13 @@ describe("memory index", () => { }; } ).db; - for (const table of ["files", "chunks", "embedding_cache", "chunks_fts", "chunks_vec"]) { + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { const existingTable = db .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") .get(table); @@ -452,18 +458,18 @@ describe("memory index", () => { run: (...params: unknown[]) => void; }; }; - const metaRow = db.prepare("SELECT value FROM meta WHERE key = ?").get("memory_index_meta_v1"); + const metaRow = db + .prepare("SELECT value FROM memory_index_meta WHERE key = ?") + .get("memory_index_meta_v1"); const meta = JSON.parse(metaRow?.value ?? "{}") as MemoryIndexMeta; - db.prepare("UPDATE meta SET value = ? WHERE key = ?").run( + db.prepare("UPDATE memory_index_meta SET value = ? WHERE key = ?").run( JSON.stringify({ ...meta, model, providerKey }), "memory_index_meta_v1", ); - db.prepare("UPDATE chunks SET model = ?").run(model); - db.prepare("UPDATE embedding_cache SET model = ?, provider_key = ? WHERE provider = ?").run( - model, - providerKey, - identityAliasFixture.provider, - ); + db.prepare("UPDATE memory_index_chunks SET model = ?").run(model); + db.prepare( + "UPDATE memory_embedding_cache SET model = ?, provider_key = ? WHERE provider = ?", + ).run(model, providerKey, identityAliasFixture.provider); } async function expectHybridKeywordSearchFindsMemory(cfg: TestCfg) { @@ -634,7 +640,7 @@ describe("memory index", () => { db: { prepare: (sql: string) => { get: (...args: unknown[]) => unknown } }; } ).db - .prepare("SELECT embedding FROM chunks WHERE path LIKE ? AND source = ?") + .prepare("SELECT embedding FROM memory_index_chunks WHERE path LIKE ? AND source = ?") .get("%2026-01-13.md", "memory") as { embedding: string } | undefined; expect(betaRow).toBeDefined(); @@ -1009,7 +1015,7 @@ describe("memory index", () => { nextManager as unknown as { db: { exec: (sql: string) => void }; } - ).db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`); + ).db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`); expect(nextManager.status().custom?.indexIdentity).toEqual({ status: "missing", reason: "index metadata is missing", @@ -1086,7 +1092,7 @@ describe("memory index", () => { }; } ).db; - db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`); + db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`); await nextManager.sync({ reason: "test" }); @@ -1095,7 +1101,7 @@ describe("memory index", () => { status: "missing", reason: "index metadata is missing", }); - const row = db.prepare("SELECT model FROM chunks LIMIT 1").get(); + const row = db.prepare("SELECT model FROM memory_index_chunks LIMIT 1").get(); expect(row?.model).toBe("semantic-embed"); } finally { await nextManager.close?.(); @@ -1197,7 +1203,7 @@ describe("memory index", () => { nextManager as unknown as { db: { exec: (sql: string) => void }; } - ).db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`); + ).db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`); const status = nextManager.status(); @@ -1627,7 +1633,10 @@ describe("memory index", () => { const originalPrepare = db.prepare.bind(db); let ftsSelects = 0; const prepareSpy = vi.spyOn(db, "prepare").mockImplementation((sql: string) => { - if (sql.includes("FROM chunks_fts") && sql.includes("WHERE chunks_fts MATCH ?")) { + if ( + sql.includes("FROM memory_index_chunks_fts") && + sql.includes("WHERE memory_index_chunks_fts MATCH ?") + ) { ftsSelects += 1; } return originalPrepare(sql); diff --git a/extensions/memory-core/src/memory/manager-db.test.ts b/extensions/memory-core/src/memory/manager-db.test.ts index 4caaa1f8b5d1..b0535f891654 100644 --- a/extensions/memory-core/src/memory/manager-db.test.ts +++ b/extensions/memory-core/src/memory/manager-db.test.ts @@ -15,12 +15,10 @@ import { } from "./manager-db.js"; import { acquireMemoryReindexLock } from "./manager-reindex-lock.js"; -function ensureTestMemorySchema(db: DatabaseSync): void { +function ensureTestMemorySchema(db: DatabaseSync, cacheEnabled = true): void { ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", - cacheEnabled: true, - ftsTable: "chunks_fts", + cacheEnabled, ftsEnabled: false, }); } @@ -48,8 +46,10 @@ describe("memory manager database publication", () => { try { ensureTestMemorySchema(targetDb); ensureTestMemorySchema(sourceDb); - targetDb.exec("CREATE TABLE chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)"); - targetDb.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)").run("stale", "[]"); + targetDb.exec("CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)"); + targetDb + .prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)") + .run("stale", "[]"); sourceDb.close(); await publishMemoryDatabaseTables({ @@ -61,7 +61,9 @@ describe("memory manager database publication", () => { expect( targetDb - .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_vec'") + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_index_chunks_vec'", + ) .get(), ).toBeUndefined(); } finally { @@ -85,13 +87,13 @@ describe("memory manager database publication", () => { return; } sourceDb.exec(` - CREATE VIRTUAL TABLE chunks_vec USING vec0( + CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0( id TEXT PRIMARY KEY, embedding FLOAT[3] ) `); sourceDb - .prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)") + .prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)") .run("vector", JSON.stringify([0, 1, 0])); sourceDb.close(); @@ -103,7 +105,9 @@ describe("memory manager database publication", () => { vectorExtensionPath: sourceVector.extensionPath, }); - expect(targetDb.prepare("SELECT id FROM chunks_vec").all()).toEqual([{ id: "vector" }]); + expect(targetDb.prepare("SELECT id FROM memory_index_chunks_vec").all()).toEqual([ + { id: "vector" }, + ]); } finally { try { sourceDb.close(); @@ -122,16 +126,22 @@ describe("memory manager database publication", () => { ensureTestMemorySchema(targetDb); ensureTestMemorySchema(sourceDb); targetDb - .prepare("INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)") + .prepare( + "INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)", + ) .run("memory.md", "memory", "published", 1, 1); sourceDb - .prepare("INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)") + .prepare( + "INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)", + ) .run("memory.md", "memory", "shadow", 1, 1); const expectedRevision = readMemoryDatabaseRevision(targetDb); sourceDb.close(); concurrentDb = new DatabaseSync(targetPath); - concurrentDb.prepare("UPDATE files SET hash = ? WHERE path = ?").run("newer", "memory.md"); + concurrentDb + .prepare("UPDATE memory_index_sources SET hash = ? WHERE path = ?") + .run("newer", "memory.md"); concurrentDb.close(); concurrentDb = undefined; @@ -144,7 +154,7 @@ describe("memory manager database publication", () => { }), ).rejects.toThrow(/changed while full reindex was building/); expect( - targetDb.prepare("SELECT hash FROM files WHERE path = ?").get("memory.md"), + targetDb.prepare("SELECT hash FROM memory_index_sources WHERE path = ?").get("memory.md"), ).toEqual({ hash: "newer" }); } finally { try { @@ -157,6 +167,41 @@ describe("memory manager database publication", () => { } }); + it("preserves the live embedding cache when the shadow index has caching disabled", async () => { + const targetPath = path.join(fixtureRoot, "target.sqlite"); + const sourcePath = path.join(fixtureRoot, "source.sqlite"); + const targetDb = new DatabaseSync(targetPath); + const sourceDb = new DatabaseSync(sourcePath); + try { + ensureTestMemorySchema(targetDb); + ensureTestMemorySchema(sourceDb, false); + targetDb + .prepare( + `INSERT INTO memory_embedding_cache ( + provider, model, provider_key, hash, embedding, dims, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run("test", "model", "key", "hash", "[]", 0, 1); + sourceDb.close(); + + await publishMemoryDatabaseTables({ + targetDb, + sourcePath, + metaKey: "memory_index_meta", + expectedRevision: readMemoryDatabaseRevision(targetDb), + }); + + expect(targetDb.prepare("SELECT hash FROM memory_embedding_cache").all()).toEqual([ + { hash: "hash" }, + ]); + } finally { + try { + sourceDb.close(); + } catch {} + targetDb.close(); + } + }); + it("removes aged orphan shadows but preserves young and locked shadows", async () => { const databasePath = path.join(fixtureRoot, "agent.sqlite"); const database = new DatabaseSync(databasePath); diff --git a/extensions/memory-core/src/memory/manager-db.ts b/extensions/memory-core/src/memory/manager-db.ts index 7c4d3bfaae62..221093ab6825 100644 --- a/extensions/memory-core/src/memory/manager-db.ts +++ b/extensions/memory-core/src/memory/manager-db.ts @@ -91,7 +91,7 @@ export function readMemoryDatabaseRevision(db: DatabaseSync): number { function replaceVirtualTable(params: { db: DatabaseSync; - tableName: "chunks_fts" | "chunks_vec"; + tableName: "memory_index_chunks_fts" | "memory_index_chunks_vec"; columns: string; ignoreDropErrorWhenSourceMissing?: boolean; }): void { @@ -126,7 +126,7 @@ export async function publishMemoryDatabaseTables(params: { params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath); try { if ( - tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "chunks_vec") && + tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_index_chunks_vec") && !hasSqliteVecExtension(params.targetDb) ) { const loaded = await loadSqliteVecExtension({ @@ -148,47 +148,49 @@ export async function publishMemoryDatabaseTables(params: { `(expected revision ${params.expectedRevision}, found ${liveRevision}); retry the full reindex.`, ); } - params.targetDb.prepare("DELETE FROM main.meta WHERE key = ?").run(params.metaKey); + params.targetDb + .prepare("DELETE FROM main.memory_index_meta WHERE key = ?") + .run(params.metaKey); params.targetDb .prepare( - `INSERT INTO main.meta (key, value) - SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.meta WHERE key = ?`, + `INSERT INTO main.memory_index_meta (key, value) + SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_meta WHERE key = ?`, ) .run(params.metaKey); params.targetDb.exec(` - DELETE FROM main.files; - INSERT INTO main.files (path, source, hash, mtime, size) - SELECT path, source, hash, mtime, size FROM ${MEMORY_REINDEX_SCHEMA}.files; + DELETE FROM main.memory_index_sources; + INSERT INTO main.memory_index_sources (path, source, hash, mtime, size) + SELECT path, source, hash, mtime, size FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_sources; - DELETE FROM main.chunks; - INSERT INTO main.chunks ( + DELETE FROM main.memory_index_chunks; + INSERT INTO main.memory_index_chunks ( id, path, source, start_line, end_line, hash, model, text, embedding, updated_at ) SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at - FROM ${MEMORY_REINDEX_SCHEMA}.chunks; + FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunks; `); - if (tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "embedding_cache")) { + if (tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_embedding_cache")) { params.targetDb.exec(` - DELETE FROM main.embedding_cache; - INSERT INTO main.embedding_cache ( + DELETE FROM main.memory_embedding_cache; + INSERT INTO main.memory_embedding_cache ( provider, model, provider_key, hash, embedding, dims, updated_at ) SELECT provider, model, provider_key, hash, embedding, dims, updated_at - FROM ${MEMORY_REINDEX_SCHEMA}.embedding_cache; + FROM ${MEMORY_REINDEX_SCHEMA}.memory_embedding_cache; `); } replaceVirtualTable({ db: params.targetDb, - tableName: "chunks_fts", + tableName: "memory_index_chunks_fts", columns: "text, id, path, source, model, start_line, end_line", }); replaceVirtualTable({ db: params.targetDb, - tableName: "chunks_vec", + tableName: "memory_index_chunks_vec", columns: "id, embedding", // A vector-disabled connection may not have sqlite-vec loaded and cannot // drop an old virtual table. Missing vector metadata forces a strict diff --git a/extensions/memory-core/src/memory/manager-embedding-cache.test.ts b/extensions/memory-core/src/memory/manager-embedding-cache.test.ts index 1422f3d74c4c..3c5d68240762 100644 --- a/extensions/memory-core/src/memory/manager-embedding-cache.test.ts +++ b/extensions/memory-core/src/memory/manager-embedding-cache.test.ts @@ -17,9 +17,7 @@ describe("memory embedding cache", () => { const db = new DatabaseSync(":memory:"); ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: true, - ftsTable: "chunks_fts", ftsEnabled: false, ftsTokenizer: "unicode61", }); diff --git a/extensions/memory-core/src/memory/manager-embedding-cache.ts b/extensions/memory-core/src/memory/manager-embedding-cache.ts index c98f76ab984f..3cd04153a3c4 100644 --- a/extensions/memory-core/src/memory/manager-embedding-cache.ts +++ b/extensions/memory-core/src/memory/manager-embedding-cache.ts @@ -36,7 +36,7 @@ export function loadMemoryEmbeddingCache(params: { return new Map(); } - const tableName = params.tableName ?? "embedding_cache"; + const tableName = params.tableName ?? "memory_embedding_cache"; const out = new Map(); const batchSize = 400; for (const identity of params.providerIdentities) { @@ -73,7 +73,7 @@ export function upsertMemoryEmbeddingCache(params: { if (!params.enabled || !provider || !params.providerKey || params.entries.length === 0) { return; } - const tableName = params.tableName ?? "embedding_cache"; + const tableName = params.tableName ?? "memory_embedding_cache"; const now = params.now ?? Date.now(); const stmt = params.db.prepare( `INSERT INTO ${tableName} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` + diff --git a/extensions/memory-core/src/memory/manager-embedding-ops.ts b/extensions/memory-core/src/memory/manager-embedding-ops.ts index 9d8423111034..c3b96f1f1409 100644 --- a/extensions/memory-core/src/memory/manager-embedding-ops.ts +++ b/extensions/memory-core/src/memory/manager-embedding-ops.ts @@ -12,6 +12,9 @@ import { buildMultimodalChunkForIndexing, chunkMarkdown, hashText, + MEMORY_EMBEDDING_CACHE_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_VECTOR_TABLE, remapChunkLines, retryTransientMemoryRead, runWithConcurrency, @@ -49,9 +52,9 @@ import { MemoryManagerSyncOps, type MemoryIndexWorkItem } from "./manager-sync-o import { logMemoryVectorDegradedWrite } from "./manager-vector-warning.js"; import { replaceMemoryVectorRow } from "./manager-vector-write.js"; -const VECTOR_TABLE = "chunks_vec"; -const FTS_TABLE = "chunks_fts"; -const EMBEDDING_CACHE_TABLE = "embedding_cache"; +const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; +const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; +const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE; const EMBEDDING_BATCH_MAX_TOKENS = 8000; const EMBEDDING_INDEX_CONCURRENCY = 4; const EMBEDDING_RETRY_MAX_ATTEMPTS = 3; @@ -690,7 +693,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { try { this.db .prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`, + `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, ) .run(pathname, source); } catch {} @@ -706,13 +709,15 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { }); } catch {} } - this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(pathname, source); + this.db + .prepare(`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`) + .run(pathname, source); } private upsertFileRecord(entry: MemoryIndexEntry, source: MemorySource): void { this.db .prepare( - `INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?) + `INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET source=excluded.source, hash=excluded.hash, @@ -723,7 +728,9 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { } private deleteFileRecord(pathname: string, source: MemorySource): void { - this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(pathname, source); + this.db + .prepare(`DELETE FROM memory_index_sources WHERE path = ? AND source = ?`) + .run(pathname, source); } /** @@ -749,7 +756,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { ); this.db .prepare( - `INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) + `INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET hash=excluded.hash, diff --git a/extensions/memory-core/src/memory/manager-fts-state.test.ts b/extensions/memory-core/src/memory/manager-fts-state.test.ts index 8a25c967e824..8de78a1f6a7b 100644 --- a/extensions/memory-core/src/memory/manager-fts-state.test.ts +++ b/extensions/memory-core/src/memory/manager-fts-state.test.ts @@ -13,23 +13,23 @@ describe("memory FTS state", () => { it("removes rows for all models when a provider is active", () => { db = new DatabaseSync(":memory:"); - db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)"); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)"); + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-12.md", "memory", "mock-embed", ); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-12.md", "memory", "other-model", ); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-13.md", "memory", "other-model", ); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-12.md", "sessions", "other-model", @@ -42,7 +42,9 @@ describe("memory FTS state", () => { currentModel: "mock-embed", }); - const rows = db.prepare("SELECT path, source, model FROM chunks_fts ORDER BY path, source").all() as Array<{ + const rows = db + .prepare("SELECT path, source, model FROM memory_index_chunks_fts ORDER BY path, source") + .all() as Array<{ path: string; source: string; model: string; @@ -55,13 +57,13 @@ describe("memory FTS state", () => { it("removes all rows for the path in FTS-only mode", () => { db = new DatabaseSync(":memory:"); - db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)"); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)"); + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-12.md", "memory", "mock-embed", ); - db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run( + db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run( "memory/2026-01-12.md", "memory", "fts-only", @@ -73,7 +75,9 @@ describe("memory FTS state", () => { source: "memory", }); - const count = db.prepare("SELECT COUNT(*) as c FROM chunks_fts").get() as { c: number }; + const count = db.prepare("SELECT COUNT(*) as c FROM memory_index_chunks_fts").get() as { + c: number; + }; expect(count.c).toBe(0); }); }); diff --git a/extensions/memory-core/src/memory/manager-fts-state.ts b/extensions/memory-core/src/memory/manager-fts-state.ts index eafa89f7e064..684f288c0e2f 100644 --- a/extensions/memory-core/src/memory/manager-fts-state.ts +++ b/extensions/memory-core/src/memory/manager-fts-state.ts @@ -9,7 +9,7 @@ export function deleteMemoryFtsRows(params: { source: MemorySource; currentModel?: string; }): void { - const tableName = params.tableName ?? "chunks_fts"; + const tableName = params.tableName ?? "memory_index_chunks_fts"; // Lexical search is model-agnostic, so refreshed/deleted files must not // leave old-model FTS rows behind for the same path/source. params.db diff --git a/extensions/memory-core/src/memory/manager-search.test.ts b/extensions/memory-core/src/memory/manager-search.test.ts index 70181d515f3a..562e53c0b77e 100644 --- a/extensions/memory-core/src/memory/manager-search.test.ts +++ b/extensions/memory-core/src/memory/manager-search.test.ts @@ -25,7 +25,7 @@ function insertKeywordFixture( }, ): void { db.prepare( - "INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ).run( params.id, params.path, @@ -39,7 +39,7 @@ function insertKeywordFixture( Date.now(), ); db.prepare( - "INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run( params.text, params.id, @@ -59,9 +59,7 @@ describe("searchKeyword trigram fallback", () => { try { const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, ftsTokenizer: "trigram", }); @@ -75,9 +73,7 @@ describe("searchKeyword trigram fallback", () => { const db = new DatabaseSync(":memory:"); const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, ftsTokenizer: "trigram", }); @@ -108,7 +104,7 @@ describe("searchKeyword trigram fallback", () => { } return await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: params.query, ftsTokenizer: "trigram", limit: 10, @@ -232,9 +228,7 @@ describe("searchKeyword FTS MATCH fallback", () => { try { const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, }); return result.ftsAvailable; @@ -247,9 +241,7 @@ describe("searchKeyword FTS MATCH fallback", () => { const db = new DatabaseSync(":memory:"); const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, }); if (!result.ftsAvailable) { @@ -288,7 +280,7 @@ describe("searchKeyword FTS MATCH fallback", () => { const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Agent", ftsTokenizer: "unicode61", limit: 10, @@ -323,7 +315,7 @@ describe("searchKeyword FTS MATCH fallback", () => { const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Transformer", ftsTokenizer: "unicode61", limit: 10, @@ -368,7 +360,7 @@ describe("searchKeyword FTS MATCH fallback", () => { const brokenBuildFtsQuery = () => "BROKEN <<<"; const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Agent", ftsTokenizer: "unicode61", limit: 10, @@ -415,7 +407,7 @@ describe("searchKeyword FTS MATCH fallback", () => { const brokenBuildFtsQuery = () => "BROKEN <<<"; const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Agent cron", ftsTokenizer: "unicode61", limit: 10, @@ -449,7 +441,7 @@ describe("searchKeyword FTS MATCH fallback", () => { await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "test", ftsTokenizer: "unicode61", limit: 10, @@ -482,9 +474,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { try { const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, }); return result.ftsAvailable; @@ -500,9 +490,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { try { const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, }); if (!result.ftsAvailable) { @@ -529,7 +517,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Clyde", ftsTokenizer: "unicode61", limit: 10, @@ -550,9 +538,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { try { const result = ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: true, }); if (!result.ftsAvailable) { @@ -568,7 +554,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { endLine: 3, }); db.prepare( - "INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run( "Deleted Clyde notes from an older model", "orphan-clyde", @@ -581,7 +567,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => { const results = await searchKeyword({ db, - ftsTable: "chunks_fts", + ftsTable: "memory_index_chunks_fts", query: "Clyde", ftsTokenizer: "unicode61", limit: 10, @@ -642,7 +628,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db: { prepare } as unknown as Parameters[0]["db"], - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec: [1, 0], limit: 2, @@ -667,14 +653,12 @@ describe("searchVector sqlite-vec KNN", () => { try { ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: false, }); const insertChunk = db.prepare( - "INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ); // Just over 3x the yield batch (FALLBACK_VECTOR_BATCH_SIZE=256), so we // expect at least 3 yield points to fire during the scan. @@ -708,7 +692,7 @@ describe("searchVector sqlite-vec KNN", () => { try { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "yield-model", queryVec: [1, 0], limit: 4, @@ -736,9 +720,7 @@ describe("searchVector sqlite-vec KNN", () => { const db = new DatabaseSync(":memory:"); ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: false, }); return db; @@ -753,7 +735,7 @@ describe("searchVector sqlite-vec KNN", () => { }, ): void { db.prepare( - "INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ).run( params.id, `memory/${params.id}.md`, @@ -775,7 +757,7 @@ describe("searchVector sqlite-vec KNN", () => { insertFallbackChunk(db, { id: "other-only", model: "other-model", vector: [1, 0] }); const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec: [1, 0], limit: 5, @@ -799,7 +781,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "canonical-model", providerModelAliases: ["/cache/default.gguf"], queryVec: [1, 0], @@ -824,7 +806,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "", queryVec: [1, 0], limit: 5, @@ -846,7 +828,7 @@ describe("searchVector sqlite-vec KNN", () => { insertFallbackChunk(db, { id: "lone", model: "target-model", vector: [1, 0] }); const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec: [1, 0], limit: 5, @@ -878,7 +860,7 @@ describe("searchVector sqlite-vec KNN", () => { } const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec: [1, 0], limit: 3, @@ -941,7 +923,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec, limit, @@ -999,7 +981,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", queryVec: [1, 0], limit: 2, @@ -1026,22 +1008,22 @@ describe("searchVector sqlite-vec KNN", () => { expect(loaded.ok, loaded.error).toBe(true); ensureMemoryIndexSchema({ db, - embeddingCacheTable: "embedding_cache", cacheEnabled: false, - ftsTable: "chunks_fts", ftsEnabled: false, }); db.exec(` - CREATE VIRTUAL TABLE chunks_vec USING vec0( + CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0( id TEXT PRIMARY KEY, embedding FLOAT[2] ); `); const insertChunk = db.prepare( - "INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ); + const insertVector = db.prepare( + "INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)", ); - const insertVector = db.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)"); const addChunk = (params: { id: string; model: string; vector: [number, number] }) => { insertChunk.run( params.id, @@ -1067,7 +1049,7 @@ describe("searchVector sqlite-vec KNN", () => { const results = await searchVector({ db, - vectorTable: "chunks_vec", + vectorTable: "memory_index_chunks_vec", providerModel: "target-model", providerModelAliases: ["alias-model"], queryVec: [1, 0], diff --git a/extensions/memory-core/src/memory/manager-search.ts b/extensions/memory-core/src/memory/manager-search.ts index 62a9b3681e2b..fbd029460208 100644 --- a/extensions/memory-core/src/memory/manager-search.ts +++ b/extensions/memory-core/src/memory/manager-search.ts @@ -159,7 +159,7 @@ export async function searchVector(params: { // which runs in ~O(log N + k) via the vec0 index, instead of the previous // full-table scan over vec_distance_cosine(). Keep vec_distance_cosine() in // the SELECT so `score = 1 - dist` stays in the cosine [0, 1] range the - // downstream merge/minScore pipeline expects. (chunks_vec is created with + // downstream merge/minScore pipeline expects. (memory_index_chunks_vec is created with // sqlite-vec's default L2 distance, so v.distance cannot be used directly // for scoring.) const qBlob = vectorToBlob(params.queryVec); @@ -170,7 +170,7 @@ export async function searchVector(params: { ` c.source,\n` + ` vec_distance_cosine(v.embedding, ?) AS dist\n` + ` FROM ${params.vectorTable} v\n` + - ` JOIN chunks c ON c.id = v.id\n` + + ` JOIN memory_index_chunks c ON c.id = v.id\n` + ` WHERE v.embedding MATCH ? AND k = ? AND ${vectorModelFilter}${params.sourceFilterVec.sql}\n` + ` ORDER BY dist ASC\n` + ` LIMIT ?`, @@ -198,7 +198,7 @@ export async function searchVector(params: { const matchingChunkCount = readCount( params.db .prepare( - `SELECT COUNT(*) AS count FROM chunks c WHERE ${vectorModelFilter}${params.sourceFilterVec.sql}`, + `SELECT COUNT(*) AS count FROM memory_index_chunks c WHERE ${vectorModelFilter}${params.sourceFilterVec.sql}`, ) .get(...providerModels, ...params.sourceFilterVec.params) as | { count?: number | bigint } @@ -257,7 +257,7 @@ async function searchChunksByEmbedding(params: { // below. The rowid cursor keeps memory bounded without OFFSET rescans. const stmt = params.db.prepare( `SELECT rowid, id, path, start_line, end_line, text, embedding, source\n` + - ` FROM chunks\n` + + ` FROM memory_index_chunks\n` + ` WHERE ${modelFilter} AND rowid > ?${params.sourceFilter.sql}\n` + ` ORDER BY rowid ASC\n` + ` LIMIT ?`, @@ -348,7 +348,7 @@ export async function searchKeyword(params: { // Lexical FTS is model-agnostic (issue #48300), but old databases may // already contain orphaned FTS rows from prior model-scoped cleanup. - const liveChunkClause = ` AND EXISTS (SELECT 1 FROM chunks c WHERE c.id = ${params.ftsTable}.id)`; + const liveChunkClause = ` AND EXISTS (SELECT 1 FROM memory_index_chunks c WHERE c.id = ${params.ftsTable}.id)`; const substringClause = plan.substringTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join(""); const substringParams = plan.substringTerms.map((term) => `%${escapeLikePattern(term)}%`); diff --git a/extensions/memory-core/src/memory/manager-source-state.ts b/extensions/memory-core/src/memory/manager-source-state.ts index a918f6b9203f..235cada788f0 100644 --- a/extensions/memory-core/src/memory/manager-source-state.ts +++ b/extensions/memory-core/src/memory/manager-source-state.ts @@ -16,8 +16,8 @@ type MemorySourceStateDb = { }; }; -export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash, mtime, size FROM files WHERE source = ?`; -export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM files WHERE path = ? AND source = ?`; +export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash, mtime, size FROM memory_index_sources WHERE source = ?`; +export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM memory_index_sources WHERE path = ? AND source = ?`; export function loadMemorySourceFileState(params: { db: MemorySourceStateDb; diff --git a/extensions/memory-core/src/memory/manager-status-state.ts b/extensions/memory-core/src/memory/manager-status-state.ts index 9f36b42364ac..2780114581ab 100644 --- a/extensions/memory-core/src/memory/manager-status-state.ts +++ b/extensions/memory-core/src/memory/manager-status-state.ts @@ -20,9 +20,9 @@ type StatusAggregateDb = { }; export const MEMORY_STATUS_AGGREGATE_SQL = - `SELECT 'files' AS kind, source, COUNT(*) as c FROM files WHERE 1=1__FILTER__ GROUP BY source\n` + + `SELECT 'files' AS kind, source, COUNT(*) as c FROM memory_index_sources WHERE 1=1__FILTER__ GROUP BY source\n` + `UNION ALL\n` + - `SELECT 'chunks' AS kind, source, COUNT(*) as c FROM chunks WHERE 1=1__FILTER__ GROUP BY source`; + `SELECT 'chunks' AS kind, source, COUNT(*) as c FROM memory_index_chunks WHERE 1=1__FILTER__ GROUP BY source`; export function resolveInitialMemoryDirty(params: { hasMemorySource: boolean; diff --git a/extensions/memory-core/src/memory/manager-sync-ops.ts b/extensions/memory-core/src/memory/manager-sync-ops.ts index 69d253b8f78f..eac5cacbca97 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.ts @@ -29,6 +29,9 @@ import { isFileMissingError, listMemoryFiles, loadSqliteVecExtension, + MEMORY_EMBEDDING_CACHE_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_VECTOR_TABLE, normalizeExtraMemoryPaths, retryTransientMemoryRead, runWithConcurrency, @@ -141,9 +144,9 @@ type MemoryReindexRetryState = { }; const META_KEY = "memory_index_meta_v1"; -const VECTOR_TABLE = "chunks_vec"; -const FTS_TABLE = "chunks_fts"; -const EMBEDDING_CACHE_TABLE = "embedding_cache"; +const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; +const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; +const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE; const SESSION_DIRTY_DEBOUNCE_MS = 5000; const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024; const SESSION_SYNC_YIELD_EVERY = 10; @@ -477,7 +480,7 @@ export abstract class MemoryManagerSyncOps { } protected hasIndexedChunks(): boolean { - const row = this.db.prepare(`SELECT 1 as found FROM chunks LIMIT 1`).get() as + const row = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as | { found?: number } | undefined; return row?.found === 1; @@ -485,7 +488,7 @@ export abstract class MemoryManagerSyncOps { protected hasSemanticChunks(): boolean { const row = this.db - .prepare(`SELECT 1 as found FROM chunks WHERE model != 'fts-only' LIMIT 1`) + .prepare(`SELECT 1 as found FROM memory_index_chunks WHERE model != 'fts-only' LIMIT 1`) .get() as { found?: number } | undefined; return row?.found === 1; } @@ -760,9 +763,7 @@ export abstract class MemoryManagerSyncOps { protected ensureSchema() { const result = ensureMemoryIndexSchema({ db: this.db, - embeddingCacheTable: EMBEDDING_CACHE_TABLE, cacheEnabled: this.cache.enabled, - ftsTable: FTS_TABLE, ftsEnabled: this.fts.enabled, ftsTokenizer: this.settings.store.fts.tokenizer, }); @@ -1743,15 +1744,15 @@ export abstract class MemoryManagerSyncOps { deferIndex?: boolean; }): Promise { const deleteFileByPathAndSource = this.db.prepare( - `DELETE FROM files WHERE path = ? AND source = ?`, + `DELETE FROM memory_index_sources WHERE path = ? AND source = ?`, ); const deleteChunksByPathAndSource = this.db.prepare( - `DELETE FROM chunks WHERE path = ? AND source = ?`, + `DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`, ); const deleteVectorRowsByPathAndSource = this.vector.enabled && this.vector.available ? this.db.prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`, + `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, ) : null; const deleteFtsRowsByPathAndSource = @@ -1873,15 +1874,15 @@ export abstract class MemoryManagerSyncOps { prefixIndexItems?: MemoryIndexWorkItem[]; }): Promise { const deleteFileByPathAndSource = this.db.prepare( - `DELETE FROM files WHERE path = ? AND source = ?`, + `DELETE FROM memory_index_sources WHERE path = ? AND source = ?`, ); const deleteChunksByPathAndSource = this.db.prepare( - `DELETE FROM chunks WHERE path = ? AND source = ?`, + `DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`, ); const deleteVectorRowsByPathAndSource = this.vector.enabled && this.vector.available ? this.db.prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`, + `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, ) : null; const deleteFtsRowsByPathAndSource = @@ -2577,9 +2578,9 @@ export abstract class MemoryManagerSyncOps { } protected readMeta(): MemoryIndexMeta | null { - const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as - | { value: string } - | undefined; + const row = this.db + .prepare(`SELECT value FROM memory_index_meta WHERE key = ?`) + .get(META_KEY) as { value: string } | undefined; if (!row?.value) { this.lastMetaSerialized = null; return null; @@ -2601,7 +2602,7 @@ export abstract class MemoryManagerSyncOps { } this.db .prepare( - `INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, + `INSERT INTO memory_index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`, ) .run(META_KEY, value); this.lastMetaSerialized = value; diff --git a/extensions/memory-core/src/memory/manager-vector-warning.test.ts b/extensions/memory-core/src/memory/manager-vector-warning.test.ts index 25e26954bd6f..ff13a02c32d2 100644 --- a/extensions/memory-core/src/memory/manager-vector-warning.test.ts +++ b/extensions/memory-core/src/memory/manager-vector-warning.test.ts @@ -27,7 +27,7 @@ describe("memory vector degradation warnings", () => { expect(second).toBe(true); expect(warn).toHaveBeenCalledTimes(1); expect(warn).toHaveBeenCalledWith( - "chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.", + "memory_index_chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.", ); }); @@ -44,7 +44,7 @@ describe("memory vector degradation warnings", () => { expect(shown).toBe(true); expect(warn).toHaveBeenCalledWith( - "chunks_vec not updated — semantic vector embeddings unavailable — no vector dimensions resolved. Vector recall degraded. Further duplicate warnings suppressed.", + "memory_index_chunks_vec not updated — semantic vector embeddings unavailable — no vector dimensions resolved. Vector recall degraded. Further duplicate warnings suppressed.", ); }); diff --git a/extensions/memory-core/src/memory/manager-vector-warning.ts b/extensions/memory-core/src/memory/manager-vector-warning.ts index e514a13b136d..d1015d0df256 100644 --- a/extensions/memory-core/src/memory/manager-vector-warning.ts +++ b/extensions/memory-core/src/memory/manager-vector-warning.ts @@ -22,7 +22,7 @@ export function logMemoryVectorDegradedWrite(params: { return params.warningShown; } params.warn( - `chunks_vec not updated — ${formatMemoryVectorDegradedWriteReason(params.loadError)}. Vector recall degraded. Further duplicate warnings suppressed.`, + `memory_index_chunks_vec not updated — ${formatMemoryVectorDegradedWriteReason(params.loadError)}. Vector recall degraded. Further duplicate warnings suppressed.`, ); return true; } diff --git a/extensions/memory-core/src/memory/manager-vector-write.ts b/extensions/memory-core/src/memory/manager-vector-write.ts index c68c13bb68d2..56e5bd95ecea 100644 --- a/extensions/memory-core/src/memory/manager-vector-write.ts +++ b/extensions/memory-core/src/memory/manager-vector-write.ts @@ -16,7 +16,7 @@ export function replaceMemoryVectorRow(params: { embedding: number[]; tableName?: string; }): void { - const tableName = params.tableName ?? "chunks_vec"; + const tableName = params.tableName ?? "memory_index_chunks_vec"; try { params.db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(params.id); } catch {} diff --git a/extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts b/extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts index 75a5b0648204..c95be90d3138 100644 --- a/extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts +++ b/extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts @@ -101,7 +101,7 @@ describe("memory manager FTS-only reindex", () => { const db = new DatabaseSync(indexPath); try { const row = db - .prepare(`SELECT COUNT(*) as c FROM chunks WHERE text LIKE ?`) + .prepare(`SELECT COUNT(*) as c FROM memory_index_chunks WHERE text LIKE ?`) .get(`%${term}%`) as { c: number } | undefined; return row?.c ?? 0; } finally { diff --git a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts index c8e4ae9d6cb9..821656012121 100644 --- a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts +++ b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts @@ -179,7 +179,7 @@ describe("memory manager reindex recovery", () => { const harness = memoryManager as unknown as ReindexHarness; const publishedRows = harness.db - .prepare("SELECT path, text FROM chunks ORDER BY path, start_line") + .prepare("SELECT path, text FROM memory_index_chunks ORDER BY path, start_line") .all(); expect(publishedRows.length).toBeGreaterThan(0); @@ -192,7 +192,9 @@ describe("memory manager reindex recovery", () => { "late shadow failure", ); expect( - harness.db.prepare("SELECT path, text FROM chunks ORDER BY path, start_line").all(), + harness.db + .prepare("SELECT path, text FROM memory_index_chunks ORDER BY path, start_line") + .all(), ).toEqual(publishedRows); }); @@ -245,7 +247,7 @@ describe("memory manager reindex recovery", () => { const databasePath = resolveOpenClawAgentSqlitePath({ agentId: "main" }); await fs.mkdir(path.dirname(databasePath), { recursive: true }); const db = new DatabaseSync(databasePath); - db.exec("CREATE TABLE chunks (id TEXT PRIMARY KEY)"); + db.exec("CREATE TABLE memory_index_chunks (id TEXT PRIMARY KEY)"); db.close(); const { getMemorySearchManager } = await import("./index.js"); @@ -275,7 +277,7 @@ describe("memory manager reindex recovery", () => { harness.db .prepare( - `INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) + `INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( diff --git a/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts b/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts index 0360949f73c4..9dfc4b8dd24a 100644 --- a/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts +++ b/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts @@ -104,8 +104,8 @@ describe("memory manager self-heal missing identity with FTS-only chunks", () => await fs.mkdir(path.dirname(indexPath), { recursive: true }); const db = new DatabaseSync(indexPath); db.exec(` - CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); - CREATE TABLE IF NOT EXISTS chunks ( + CREATE TABLE IF NOT EXISTS memory_index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS memory_index_chunks ( id TEXT PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', @@ -117,16 +117,16 @@ describe("memory manager self-heal missing identity with FTS-only chunks", () => embedding TEXT NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS files ( + CREATE TABLE IF NOT EXISTS memory_index_sources ( path TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, size INTEGER NOT NULL ); - INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) + INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES ('chunk-1', 'MEMORY.md', 'memory', 1, 3, 'hash-1', '${model}', 'Alpha topic keep note', '[]', ${Date.now()}); - INSERT INTO files (path, source, hash, mtime, size) + INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES ('MEMORY.md', 'memory', 'hash-1', ${Date.now()}, 100); `); db.close(); diff --git a/extensions/memory-core/src/memory/manager.ts b/extensions/memory-core/src/memory/manager.ts index 7b7673de6833..ece5789ccbb9 100644 --- a/extensions/memory-core/src/memory/manager.ts +++ b/extensions/memory-core/src/memory/manager.ts @@ -14,6 +14,9 @@ import { import { extractKeywords } from "openclaw/plugin-sdk/memory-core-host-engine-qmd"; import { readMemoryFile, + MEMORY_EMBEDDING_CACHE_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_VECTOR_TABLE, type MemoryEmbeddingProbeResult, type MemoryProviderStatus, type MemorySearchManager, @@ -66,9 +69,9 @@ import { } from "./manager-sync-control.js"; import { applyTemporalDecayToHybridResults } from "./temporal-decay.js"; const SNIPPET_MAX_CHARS = 700; -const VECTOR_TABLE = "chunks_vec"; -const FTS_TABLE = "chunks_fts"; -const EMBEDDING_CACHE_TABLE = "embedding_cache"; +const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; +const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; +const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE; const MEMORY_INDEX_MANAGER_CACHE_KEY = Symbol.for("openclaw.memoryIndexManagerCache"); export const EMBEDDING_PROBE_CACHE_TTL_MS = 30_000; const KEYWORD_FALLBACK_SEARCH_TERM_LIMIT = 6; @@ -821,7 +824,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem } private hasIndexedContent(): boolean { - const chunkRow = this.db.prepare(`SELECT 1 as found FROM chunks LIMIT 1`).get() as + const chunkRow = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as | { found?: number; } diff --git a/extensions/memory-core/src/memory/manager.vector-dedupe.test.ts b/extensions/memory-core/src/memory/manager.vector-dedupe.test.ts index f50b1b5849f3..290e8e385b35 100644 --- a/extensions/memory-core/src/memory/manager.vector-dedupe.test.ts +++ b/extensions/memory-core/src/memory/manager.vector-dedupe.test.ts @@ -13,7 +13,7 @@ describe("memory vector dedupe", () => { it("deletes existing vector rows before inserting replacements", () => { db = new DatabaseSync(":memory:"); - db.exec("CREATE TABLE chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)"); + db.exec("CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)"); replaceMemoryVectorRow({ db, @@ -23,8 +23,8 @@ describe("memory vector dedupe", () => { db.exec(` CREATE TRIGGER fail_if_vector_row_not_deleted - BEFORE INSERT ON chunks_vec - WHEN EXISTS (SELECT 1 FROM chunks_vec WHERE id = NEW.id) + BEFORE INSERT ON memory_index_chunks_vec + WHEN EXISTS (SELECT 1 FROM memory_index_chunks_vec WHERE id = NEW.id) BEGIN SELECT RAISE(FAIL, 'vector row not deleted before insert'); END; @@ -39,7 +39,9 @@ describe("memory vector dedupe", () => { ).toBeUndefined(); const row = db - .prepare("SELECT COUNT(*) as c, length(embedding) as bytes FROM chunks_vec WHERE id = ?") + .prepare( + "SELECT COUNT(*) as c, length(embedding) as bytes FROM memory_index_chunks_vec WHERE id = ?", + ) .get("chunk-1") as { c: number; bytes: number } | undefined; expect(row?.c).toBe(1); expect(row?.bytes).toBe(12); diff --git a/packages/memory-host-sdk/src/engine-storage.ts b/packages/memory-host-sdk/src/engine-storage.ts index 84bfe886e25e..fd6f9be4457e 100644 --- a/packages/memory-host-sdk/src/engine-storage.ts +++ b/packages/memory-host-sdk/src/engine-storage.ts @@ -39,7 +39,16 @@ export type { MemorySource, MemorySyncProgressUpdate, } from "./host/types.js"; -export { ensureMemoryIndexSchema } from "./host/memory-schema.js"; +export { + ensureMemoryIndexSchema, + MEMORY_EMBEDDING_CACHE_TABLE, + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_META_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + MEMORY_INDEX_STATE_TABLE, + MEMORY_INDEX_VECTOR_TABLE, +} from "./host/memory-schema.js"; export { loadSqliteVecExtension } from "./host/sqlite-vec.js"; export { closeMemorySqliteWalMaintenance, diff --git a/packages/memory-host-sdk/src/host/memory-schema.ts b/packages/memory-host-sdk/src/host/memory-schema.ts index fd24ae4ed1cc..97fad3d99246 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.ts @@ -4,32 +4,34 @@ import { formatErrorMessage } from "./error-utils.js"; // SQLite schema setup for builtin memory index, embedding cache, and FTS. -/** Ensure memory index tables and optional FTS/cache tables exist. */ +export const MEMORY_INDEX_META_TABLE = "memory_index_meta"; +export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources"; +export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks"; +export const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache"; +export const MEMORY_INDEX_STATE_TABLE = "memory_index_state"; +export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts"; +export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec"; + +/** Ensure canonical memory index tables and the optional FTS table exist. */ export function ensureMemoryIndexSchema(params: { db: DatabaseSync; - embeddingCacheTable: string; cacheEnabled: boolean; - ftsTable: string; ftsEnabled: boolean; ftsTokenizer?: "unicode61" | "trigram"; }): { ftsAvailable: boolean; ftsError?: string } { params.db.exec(` - CREATE TABLE IF NOT EXISTS meta ( + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); - `); - params.db.exec(` - CREATE TABLE IF NOT EXISTS files ( + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} ( path TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, size INTEGER NOT NULL ); - `); - params.db.exec(` - CREATE TABLE IF NOT EXISTS chunks ( + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} ( id TEXT PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', @@ -41,49 +43,52 @@ export function ensureMemoryIndexSchema(params: { embedding TEXT NOT NULL, updated_at INTEGER NOT NULL ); - `); - params.db.exec(` - CREATE TABLE IF NOT EXISTS memory_index_state ( + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} ( id INTEGER PRIMARY KEY CHECK (id = 1), revision INTEGER NOT NULL ); - INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0); + INSERT OR IGNORE INTO ${MEMORY_INDEX_STATE_TABLE} (id, revision) VALUES (1, 0); - CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_insert - AFTER INSERT ON files + CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert + AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; - CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_update - AFTER UPDATE ON files + CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update + AFTER UPDATE ON ${MEMORY_INDEX_SOURCES_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; - CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_delete - AFTER DELETE ON files + CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete + AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; - CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_insert - AFTER INSERT ON chunks + CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert + AFTER INSERT ON ${MEMORY_INDEX_CHUNKS_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; - CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_update - AFTER UPDATE ON chunks + CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update + AFTER UPDATE ON ${MEMORY_INDEX_CHUNKS_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; - CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_delete - AFTER DELETE ON chunks + CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete + AFTER DELETE ON ${MEMORY_INDEX_CHUNKS_TABLE} BEGIN - UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; + UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; + + CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path + ON ${MEMORY_INDEX_CHUNKS_TABLE}(path); + CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source + ON ${MEMORY_INDEX_CHUNKS_TABLE}(source); `); if (params.cacheEnabled) { params.db.exec(` - CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} ( + CREATE TABLE IF NOT EXISTS ${MEMORY_EMBEDDING_CACHE_TABLE} ( provider TEXT NOT NULL, model TEXT NOT NULL, provider_key TEXT NOT NULL, @@ -93,10 +98,9 @@ export function ensureMemoryIndexSchema(params: { updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) ); + CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at + ON ${MEMORY_EMBEDDING_CACHE_TABLE}(updated_at); `); - params.db.exec( - `CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`, - ); } let ftsAvailable = false; @@ -106,7 +110,7 @@ export function ensureMemoryIndexSchema(params: { const tokenizer = params.ftsTokenizer ?? "unicode61"; const tokenizeClause = tokenizer === "trigram" ? `, tokenize='trigram case_sensitive 0'` : ""; params.db.exec( - `CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n` + + `CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_FTS_TABLE} USING fts5(\n` + ` text,\n` + ` id UNINDEXED,\n` + ` path UNINDEXED,\n` + @@ -124,24 +128,5 @@ export function ensureMemoryIndexSchema(params: { } } - ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'"); - ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'"); - params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`); - params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`); - return { ftsAvailable, ...(ftsError ? { ftsError } : {}) }; } - -/** Add a missing shipped column without rebuilding existing memory tables. */ -function ensureColumn( - db: DatabaseSync, - table: "files" | "chunks", - column: string, - definition: string, -): void { - const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; - if (rows.some((row) => row.name === column)) { - return; - } - db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); -} diff --git a/src/commands/status.scan.shared.ts b/src/commands/status.scan.shared.ts index c1543bceffe3..b4c930480f30 100644 --- a/src/commands/status.scan.shared.ts +++ b/src/commands/status.scan.shared.ts @@ -18,7 +18,12 @@ import { normalizeControlUiBasePath } from "../gateway/control-ui-shared.js"; import { resolveGatewayProbeTarget } from "../gateway/probe-target.js"; import type { GatewayProbeResult, probeGateway as probeGatewayFn } from "../gateway/probe.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import type { MemoryProviderStatus } from "../memory-host-sdk/engine-storage.js"; +import { + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_META_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + type MemoryProviderStatus, +} from "../memory-host-sdk/engine-storage.js"; import { defaultSlotIdForKey } from "../plugins/slots.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; import { resolveTailscalePublishedHost } from "../shared/tailscale-status.js"; @@ -51,24 +56,29 @@ function hasBuiltInMemoryState(databasePath: string): boolean { let db: DatabaseSync | undefined; try { db = new DatabaseSync(databasePath, { readOnly: true }); + const builtInMemoryTables = [ + MEMORY_INDEX_META_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + MEMORY_INDEX_CHUNKS_TABLE, + ] as const; const tableNames = new Set( ( db - .prepare( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks')", - ) - .all() as Array<{ name?: unknown }> + .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)`) + .all(...builtInMemoryTables) as Array<{ name?: unknown }> ) .map((row) => row.name) .filter((name): name is string => typeof name === "string"), ); if ( - tableNames.has("meta") && - db.prepare("SELECT 1 AS ok FROM meta WHERE key = ? LIMIT 1").get(MEMORY_INDEX_META_KEY) + tableNames.has(MEMORY_INDEX_META_TABLE) && + db + .prepare(`SELECT 1 AS ok FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ? LIMIT 1`) + .get(MEMORY_INDEX_META_KEY) ) { return true; } - for (const tableName of ["files", "chunks"] as const) { + for (const tableName of [MEMORY_INDEX_SOURCES_TABLE, MEMORY_INDEX_CHUNKS_TABLE] as const) { if ( tableNames.has(tableName) && db.prepare(`SELECT 1 AS ok FROM ${tableName} LIMIT 1`).get() diff --git a/src/memory-host-sdk/engine-storage.ts b/src/memory-host-sdk/engine-storage.ts index 128506e08f0d..9b6784cf3372 100644 --- a/src/memory-host-sdk/engine-storage.ts +++ b/src/memory-host-sdk/engine-storage.ts @@ -3,6 +3,9 @@ * path stable while the shared SDK package owns provider status semantics. */ export { + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_META_TABLE, + MEMORY_INDEX_SOURCES_TABLE, resolveMemoryBackendConfig, type MemoryProviderStatus, } from "../../packages/memory-host-sdk/src/engine-storage.js"; diff --git a/src/plugin-sdk/memory-core-host-engine-storage.ts b/src/plugin-sdk/memory-core-host-engine-storage.ts index b5e9e807243e..91ef63272b45 100644 --- a/src/plugin-sdk/memory-core-host-engine-storage.ts +++ b/src/plugin-sdk/memory-core-host-engine-storage.ts @@ -19,6 +19,13 @@ export { isTransientMemoryReadError, listMemoryFiles, loadSqliteVecExtension, + MEMORY_EMBEDDING_CACHE_TABLE, + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_META_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + MEMORY_INDEX_STATE_TABLE, + MEMORY_INDEX_VECTOR_TABLE, normalizeExtraMemoryPaths, parseEmbedding, readMemoryFile, diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 38f5670a6361..45836a2dd0ff 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -31,6 +31,47 @@ export interface CacheEntries { value_json: string | null; } +export interface MemoryEmbeddingCache { + dims: number | null; + embedding: string; + hash: string; + model: string; + provider: string; + provider_key: string; + updated_at: number; +} + +export interface MemoryIndexChunks { + embedding: string; + end_line: number; + hash: string; + id: string | null; + model: string; + path: string; + source: Generated; + start_line: number; + text: string; + updated_at: number; +} + +export interface MemoryIndexMeta { + key: string | null; + value: string; +} + +export interface MemoryIndexSources { + hash: string; + mtime: number; + path: string | null; + size: number; + source: Generated; +} + +export interface MemoryIndexState { + id: Generated; + revision: number; +} + export interface SchemaMeta { agent_id: string | null; app_version: string | null; @@ -45,5 +86,10 @@ export interface DB { auth_profile_state: AuthProfileState; auth_profile_store: AuthProfileStore; cache_entries: CacheEntries; + memory_embedding_cache: MemoryEmbeddingCache; + memory_index_chunks: MemoryIndexChunks; + memory_index_meta: MemoryIndexMeta; + memory_index_sources: MemoryIndexSources; + memory_index_state: MemoryIndexState; schema_meta: SchemaMeta; } diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 5f5a39957a25..214c602b03ee 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -40,4 +40,93 @@ CREATE TABLE IF NOT EXISTS auth_profile_state ( state_key TEXT NOT NULL PRIMARY KEY, state_json TEXT NOT NULL, updated_at INTEGER NOT NULL -);\n`; +); + +CREATE TABLE IF NOT EXISTS memory_index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_index_sources ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_index_chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_embedding_cache ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + provider_key TEXT NOT NULL, + hash TEXT NOT NULL, + embedding TEXT NOT NULL, + dims INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, model, provider_key, hash) +); + +CREATE TABLE IF NOT EXISTS memory_index_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL +); + +INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0); + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert +AFTER INSERT ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update +AFTER UPDATE ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete +AFTER DELETE ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert +AFTER INSERT ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update +AFTER UPDATE ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete +AFTER DELETE ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at + ON memory_embedding_cache(updated_at); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path + ON memory_index_chunks(path); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source + ON memory_index_chunks(source);\n`; diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index e212a1a66a57..8c1bfa892e37 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -36,3 +36,92 @@ CREATE TABLE IF NOT EXISTS auth_profile_state ( state_json TEXT NOT NULL, updated_at INTEGER NOT NULL ); + +CREATE TABLE IF NOT EXISTS memory_index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_index_sources ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_index_chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS memory_embedding_cache ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + provider_key TEXT NOT NULL, + hash TEXT NOT NULL, + embedding TEXT NOT NULL, + dims INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, model, provider_key, hash) +); + +CREATE TABLE IF NOT EXISTS memory_index_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL +); + +INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0); + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert +AFTER INSERT ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update +AFTER UPDATE ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete +AFTER DELETE ON memory_index_sources +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert +AFTER INSERT ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update +AFTER UPDATE ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete +AFTER DELETE ON memory_index_chunks +BEGIN + UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; +END; + +CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at + ON memory_embedding_cache(updated_at); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path + ON memory_index_chunks(path); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source + ON memory_index_chunks(source);