dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-07 10:34:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(memory): canonicalize agent database tables

Browse Source
This commit is contained in:
Vincent Koc
2026-06-17 06:35:46 +02:00
parent acdcf37f87
commit 88bc08c124
29 changed files with 522 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
extensions/memory-core/src/memory/index.test.ts
View File

@@ -341,7 +341,13 @@ describe("memory index", () => {
};
}
).db;
for (const table of ["files", "chunks", "embedding_cache", "chunks_fts", "chunks_vec"]) {
for (const table of [
"memory_index_sources",
"memory_index_chunks",
"memory_embedding_cache",
"memory_index_chunks_fts",
"memory_index_chunks_vec",
]) {
const existingTable = db
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(table);
@@ -452,18 +458,18 @@ describe("memory index", () => {
run: (...params: unknown[]) => void;
};
};
const metaRow = db.prepare("SELECT value FROM meta WHERE key = ?").get("memory_index_meta_v1");
const metaRow = db
.prepare("SELECT value FROM memory_index_meta WHERE key = ?")
.get("memory_index_meta_v1");
const meta = JSON.parse(metaRow?.value ?? "{}") as MemoryIndexMeta;
db.prepare("UPDATE meta SET value = ? WHERE key = ?").run(
db.prepare("UPDATE memory_index_meta SET value = ? WHERE key = ?").run(
JSON.stringify({ ...meta, model, providerKey }),
"memory_index_meta_v1",
);
db.prepare("UPDATE chunks SET model = ?").run(model);
db.prepare("UPDATE embedding_cache SET model = ?, provider_key = ? WHERE provider = ?").run(
model,
providerKey,
identityAliasFixture.provider,
);
db.prepare("UPDATE memory_index_chunks SET model = ?").run(model);
db.prepare(
"UPDATE memory_embedding_cache SET model = ?, provider_key = ? WHERE provider = ?",
).run(model, providerKey, identityAliasFixture.provider);
}
async function expectHybridKeywordSearchFindsMemory(cfg: TestCfg) {
@@ -634,7 +640,7 @@ describe("memory index", () => {
db: { prepare: (sql: string) => { get: (...args: unknown[]) => unknown } };
}
).db
.prepare("SELECT embedding FROM chunks WHERE path LIKE ? AND source = ?")
.prepare("SELECT embedding FROM memory_index_chunks WHERE path LIKE ? AND source = ?")
.get("%2026-01-13.md", "memory") as { embedding: string } | undefined;
expect(betaRow).toBeDefined();
@@ -1009,7 +1015,7 @@ describe("memory index", () => {
nextManager as unknown as {
db: { exec: (sql: string) => void };
}
).db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`);
).db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`);
expect(nextManager.status().custom?.indexIdentity).toEqual({
status: "missing",
reason: "index metadata is missing",
@@ -1086,7 +1092,7 @@ describe("memory index", () => {
};
}
).db;
db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`);
db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`);
await nextManager.sync({ reason: "test" });
@@ -1095,7 +1101,7 @@ describe("memory index", () => {
status: "missing",
reason: "index metadata is missing",
});
const row = db.prepare("SELECT model FROM chunks LIMIT 1").get();
const row = db.prepare("SELECT model FROM memory_index_chunks LIMIT 1").get();
expect(row?.model).toBe("semantic-embed");
} finally {
await nextManager.close?.();
@@ -1197,7 +1203,7 @@ describe("memory index", () => {
nextManager as unknown as {
db: { exec: (sql: string) => void };
}
).db.exec(`DELETE FROM meta WHERE key = 'memory_index_meta_v1'`);
).db.exec(`DELETE FROM memory_index_meta WHERE key = 'memory_index_meta_v1'`);
const status = nextManager.status();
@@ -1627,7 +1633,10 @@ describe("memory index", () => {
const originalPrepare = db.prepare.bind(db);
let ftsSelects = 0;
const prepareSpy = vi.spyOn(db, "prepare").mockImplementation((sql: string) => {
if (sql.includes("FROM chunks_fts") && sql.includes("WHERE chunks_fts MATCH ?")) {
if (
sql.includes("FROM memory_index_chunks_fts") &&
sql.includes("WHERE memory_index_chunks_fts MATCH ?")
) {
ftsSelects += 1;
}
return originalPrepare(sql);

73
extensions/memory-core/src/memory/manager-db.test.ts
View File

@@ -15,12 +15,10 @@ import {
} from "./manager-db.js";
import { acquireMemoryReindexLock } from "./manager-reindex-lock.js";
function ensureTestMemorySchema(db: DatabaseSync): void {
function ensureTestMemorySchema(db: DatabaseSync, cacheEnabled = true): void {
ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: true,
ftsTable: "chunks_fts",
cacheEnabled,
ftsEnabled: false,
});
}
@@ -48,8 +46,10 @@ describe("memory manager database publication", () => {
try {
ensureTestMemorySchema(targetDb);
ensureTestMemorySchema(sourceDb);
targetDb.exec("CREATE TABLE chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
targetDb.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)").run("stale", "[]");
targetDb.exec("CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
targetDb
.prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)")
.run("stale", "[]");
sourceDb.close();
await publishMemoryDatabaseTables({
@@ -61,7 +61,9 @@ describe("memory manager database publication", () => {
expect(
targetDb
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_vec'")
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_index_chunks_vec'",
)
.get(),
).toBeUndefined();
} finally {
@@ -85,13 +87,13 @@ describe("memory manager database publication", () => {
return;
}
sourceDb.exec(`
CREATE VIRTUAL TABLE chunks_vec USING vec0(
CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[3]
)
`);
sourceDb
.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)")
.prepare("INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)")
.run("vector", JSON.stringify([0, 1, 0]));
sourceDb.close();
@@ -103,7 +105,9 @@ describe("memory manager database publication", () => {
vectorExtensionPath: sourceVector.extensionPath,
});
expect(targetDb.prepare("SELECT id FROM chunks_vec").all()).toEqual([{ id: "vector" }]);
expect(targetDb.prepare("SELECT id FROM memory_index_chunks_vec").all()).toEqual([
{ id: "vector" },
]);
} finally {
try {
sourceDb.close();
@@ -122,16 +126,22 @@ describe("memory manager database publication", () => {
ensureTestMemorySchema(targetDb);
ensureTestMemorySchema(sourceDb);
targetDb
.prepare("INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)")
.prepare(
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
)
.run("memory.md", "memory", "published", 1, 1);
sourceDb
.prepare("INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)")
.prepare(
"INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)",
)
.run("memory.md", "memory", "shadow", 1, 1);
const expectedRevision = readMemoryDatabaseRevision(targetDb);
sourceDb.close();
concurrentDb = new DatabaseSync(targetPath);
concurrentDb.prepare("UPDATE files SET hash = ? WHERE path = ?").run("newer", "memory.md");
concurrentDb
.prepare("UPDATE memory_index_sources SET hash = ? WHERE path = ?")
.run("newer", "memory.md");
concurrentDb.close();
concurrentDb = undefined;
@@ -144,7 +154,7 @@ describe("memory manager database publication", () => {
}),
).rejects.toThrow(/changed while full reindex was building/);
expect(
targetDb.prepare("SELECT hash FROM files WHERE path = ?").get("memory.md"),
targetDb.prepare("SELECT hash FROM memory_index_sources WHERE path = ?").get("memory.md"),
).toEqual({ hash: "newer" });
} finally {
try {
@@ -157,6 +167,41 @@ describe("memory manager database publication", () => {
}
});
it("preserves the live embedding cache when the shadow index has caching disabled", async () => {
const targetPath = path.join(fixtureRoot, "target.sqlite");
const sourcePath = path.join(fixtureRoot, "source.sqlite");
const targetDb = new DatabaseSync(targetPath);
const sourceDb = new DatabaseSync(sourcePath);
try {
ensureTestMemorySchema(targetDb);
ensureTestMemorySchema(sourceDb, false);
targetDb
.prepare(
`INSERT INTO memory_embedding_cache (
provider, model, provider_key, hash, embedding, dims, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
)
.run("test", "model", "key", "hash", "[]", 0, 1);
sourceDb.close();
await publishMemoryDatabaseTables({
targetDb,
sourcePath,
metaKey: "memory_index_meta",
expectedRevision: readMemoryDatabaseRevision(targetDb),
});
expect(targetDb.prepare("SELECT hash FROM memory_embedding_cache").all()).toEqual([
{ hash: "hash" },
]);
} finally {
try {
sourceDb.close();
} catch {}
targetDb.close();
}
});
it("removes aged orphan shadows but preserves young and locked shadows", async () => {
const databasePath = path.join(fixtureRoot, "agent.sqlite");
const database = new DatabaseSync(databasePath);

36
extensions/memory-core/src/memory/manager-db.ts
View File

@@ -91,7 +91,7 @@ export function readMemoryDatabaseRevision(db: DatabaseSync): number {
function replaceVirtualTable(params: {
db: DatabaseSync;
tableName: "chunks_fts" | "chunks_vec";
tableName: "memory_index_chunks_fts" | "memory_index_chunks_vec";
columns: string;
ignoreDropErrorWhenSourceMissing?: boolean;
}): void {
@@ -126,7 +126,7 @@ export async function publishMemoryDatabaseTables(params: {
params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath);
try {
if (
tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "chunks_vec") &&
tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_index_chunks_vec") &&
!hasSqliteVecExtension(params.targetDb)
) {
const loaded = await loadSqliteVecExtension({
@@ -148,47 +148,49 @@ export async function publishMemoryDatabaseTables(params: {
`(expected revision ${params.expectedRevision}, found ${liveRevision}); retry the full reindex.`,
);
}
params.targetDb.prepare("DELETE FROM main.meta WHERE key = ?").run(params.metaKey);
params.targetDb
.prepare("DELETE FROM main.memory_index_meta WHERE key = ?")
.run(params.metaKey);
params.targetDb
.prepare(
`INSERT INTO main.meta (key, value)
SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.meta WHERE key = ?`,
`INSERT INTO main.memory_index_meta (key, value)
SELECT key, value FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_meta WHERE key = ?`,
)
.run(params.metaKey);
params.targetDb.exec(`
DELETE FROM main.files;
INSERT INTO main.files (path, source, hash, mtime, size)
SELECT path, source, hash, mtime, size FROM ${MEMORY_REINDEX_SCHEMA}.files;
DELETE FROM main.memory_index_sources;
INSERT INTO main.memory_index_sources (path, source, hash, mtime, size)
SELECT path, source, hash, mtime, size FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_sources;
DELETE FROM main.chunks;
INSERT INTO main.chunks (
DELETE FROM main.memory_index_chunks;
INSERT INTO main.memory_index_chunks (
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
)
SELECT
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
FROM ${MEMORY_REINDEX_SCHEMA}.chunks;
FROM ${MEMORY_REINDEX_SCHEMA}.memory_index_chunks;
`);
if (tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "embedding_cache")) {
if (tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "memory_embedding_cache")) {
params.targetDb.exec(`
DELETE FROM main.embedding_cache;
INSERT INTO main.embedding_cache (
DELETE FROM main.memory_embedding_cache;
INSERT INTO main.memory_embedding_cache (
provider, model, provider_key, hash, embedding, dims, updated_at
)
SELECT provider, model, provider_key, hash, embedding, dims, updated_at
FROM ${MEMORY_REINDEX_SCHEMA}.embedding_cache;
FROM ${MEMORY_REINDEX_SCHEMA}.memory_embedding_cache;
`);
}
replaceVirtualTable({
db: params.targetDb,
tableName: "chunks_fts",
tableName: "memory_index_chunks_fts",
columns: "text, id, path, source, model, start_line, end_line",
});
replaceVirtualTable({
db: params.targetDb,
tableName: "chunks_vec",
tableName: "memory_index_chunks_vec",
columns: "id, embedding",
// A vector-disabled connection may not have sqlite-vec loaded and cannot
// drop an old virtual table. Missing vector metadata forces a strict

2
extensions/memory-core/src/memory/manager-embedding-cache.test.ts
View File

@@ -17,9 +17,7 @@ describe("memory embedding cache", () => {
const db = new DatabaseSync(":memory:");
ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: true,
ftsTable: "chunks_fts",
ftsEnabled: false,
ftsTokenizer: "unicode61",
});

4
extensions/memory-core/src/memory/manager-embedding-cache.ts
View File

@@ -36,7 +36,7 @@ export function loadMemoryEmbeddingCache(params: {
return new Map();
}
const tableName = params.tableName ?? "embedding_cache";
const tableName = params.tableName ?? "memory_embedding_cache";
const out = new Map<string, number[]>();
const batchSize = 400;
for (const identity of params.providerIdentities) {
@@ -73,7 +73,7 @@ export function upsertMemoryEmbeddingCache(params: {
if (!params.enabled || !provider || !params.providerKey || params.entries.length === 0) {
return;
}
const tableName = params.tableName ?? "embedding_cache";
const tableName = params.tableName ?? "memory_embedding_cache";
const now = params.now ?? Date.now();
const stmt = params.db.prepare(
`INSERT INTO ${tableName} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +

23
extensions/memory-core/src/memory/manager-embedding-ops.ts
View File

@@ -12,6 +12,9 @@ import {
buildMultimodalChunkForIndexing,
chunkMarkdown,
hashText,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
remapChunkLines,
retryTransientMemoryRead,
runWithConcurrency,
@@ -49,9 +52,9 @@ import { MemoryManagerSyncOps, type MemoryIndexWorkItem } from "./manager-sync-o
import { logMemoryVectorDegradedWrite } from "./manager-vector-warning.js";
import { replaceMemoryVectorRow } from "./manager-vector-write.js";
const VECTOR_TABLE = "chunks_vec";
const FTS_TABLE = "chunks_fts";
const EMBEDDING_CACHE_TABLE = "embedding_cache";
const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE;
const FTS_TABLE = MEMORY_INDEX_FTS_TABLE;
const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE;
const EMBEDDING_BATCH_MAX_TOKENS = 8000;
const EMBEDDING_INDEX_CONCURRENCY = 4;
const EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
@@ -690,7 +693,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
try {
this.db
.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
.run(pathname, source);
} catch {}
@@ -706,13 +709,15 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
});
} catch {}
}
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(pathname, source);
this.db
.prepare(`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`)
.run(pathname, source);
}
private upsertFileRecord(entry: MemoryIndexEntry, source: MemorySource): void {
this.db
.prepare(
`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
`INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
source=excluded.source,
hash=excluded.hash,
@@ -723,7 +728,9 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
}
private deleteFileRecord(pathname: string, source: MemorySource): void {
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(pathname, source);
this.db
.prepare(`DELETE FROM memory_index_sources WHERE path = ? AND source = ?`)
.run(pathname, source);
}
/**
@@ -749,7 +756,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
);
this.db
.prepare(
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
`INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
hash=excluded.hash,

24
extensions/memory-core/src/memory/manager-fts-state.test.ts
View File

@@ -13,23 +13,23 @@ describe("memory FTS state", () => {
it("removes rows for all models when a provider is active", () => {
db = new DatabaseSync(":memory:");
db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)");
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)");
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-12.md",
"memory",
"mock-embed",
);
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-12.md",
"memory",
"other-model",
);
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-13.md",
"memory",
"other-model",
);
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-12.md",
"sessions",
"other-model",
@@ -42,7 +42,9 @@ describe("memory FTS state", () => {
currentModel: "mock-embed",
});
const rows = db.prepare("SELECT path, source, model FROM chunks_fts ORDER BY path, source").all() as Array<{
const rows = db
.prepare("SELECT path, source, model FROM memory_index_chunks_fts ORDER BY path, source")
.all() as Array<{
path: string;
source: string;
model: string;
@@ -55,13 +57,13 @@ describe("memory FTS state", () => {
it("removes all rows for the path in FTS-only mode", () => {
db = new DatabaseSync(":memory:");
db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)");
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.exec("CREATE TABLE memory_index_chunks_fts (path TEXT, source TEXT, model TEXT)");
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-12.md",
"memory",
"mock-embed",
);
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
db.prepare("INSERT INTO memory_index_chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
"memory/2026-01-12.md",
"memory",
"fts-only",
@@ -73,7 +75,9 @@ describe("memory FTS state", () => {
source: "memory",
});
const count = db.prepare("SELECT COUNT(*) as c FROM chunks_fts").get() as { c: number };
const count = db.prepare("SELECT COUNT(*) as c FROM memory_index_chunks_fts").get() as {
c: number;
};
expect(count.c).toBe(0);
});
});

2
extensions/memory-core/src/memory/manager-fts-state.ts
View File

@@ -9,7 +9,7 @@ export function deleteMemoryFtsRows(params: {
source: MemorySource;
currentModel?: string;
}): void {
const tableName = params.tableName ?? "chunks_fts";
const tableName = params.tableName ?? "memory_index_chunks_fts";
// Lexical search is model-agnostic, so refreshed/deleted files must not
// leave old-model FTS rows behind for the same path/source.
params.db

74
extensions/memory-core/src/memory/manager-search.test.ts
View File

@@ -25,7 +25,7 @@ function insertKeywordFixture(
},
): void {
db.prepare(
"INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
).run(
params.id,
params.path,
@@ -39,7 +39,7 @@ function insertKeywordFixture(
Date.now(),
);
db.prepare(
"INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
).run(
params.text,
params.id,
@@ -59,9 +59,7 @@ describe("searchKeyword trigram fallback", () => {
try {
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
ftsTokenizer: "trigram",
});
@@ -75,9 +73,7 @@ describe("searchKeyword trigram fallback", () => {
const db = new DatabaseSync(":memory:");
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
ftsTokenizer: "trigram",
});
@@ -108,7 +104,7 @@ describe("searchKeyword trigram fallback", () => {
}
return await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: params.query,
ftsTokenizer: "trigram",
limit: 10,
@@ -232,9 +228,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
try {
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
});
return result.ftsAvailable;
@@ -247,9 +241,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
const db = new DatabaseSync(":memory:");
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
});
if (!result.ftsAvailable) {
@@ -288,7 +280,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Agent",
ftsTokenizer: "unicode61",
limit: 10,
@@ -323,7 +315,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Transformer",
ftsTokenizer: "unicode61",
limit: 10,
@@ -368,7 +360,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
const brokenBuildFtsQuery = () => "BROKEN <<<";
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Agent",
ftsTokenizer: "unicode61",
limit: 10,
@@ -415,7 +407,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
const brokenBuildFtsQuery = () => "BROKEN <<<";
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Agent cron",
ftsTokenizer: "unicode61",
limit: 10,
@@ -449,7 +441,7 @@ describe("searchKeyword FTS MATCH fallback", () => {
await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "test",
ftsTokenizer: "unicode61",
limit: 10,
@@ -482,9 +474,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
try {
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
});
return result.ftsAvailable;
@@ -500,9 +490,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
try {
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
});
if (!result.ftsAvailable) {
@@ -529,7 +517,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Clyde",
ftsTokenizer: "unicode61",
limit: 10,
@@ -550,9 +538,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
try {
const result = ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: true,
});
if (!result.ftsAvailable) {
@@ -568,7 +554,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
endLine: 3,
});
db.prepare(
"INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
).run(
"Deleted Clyde notes from an older model",
"orphan-clyde",
@@ -581,7 +567,7 @@ describe("searchKeyword cross-model FTS visibility (issue #48300)", () => {
const results = await searchKeyword({
db,
ftsTable: "chunks_fts",
ftsTable: "memory_index_chunks_fts",
query: "Clyde",
ftsTokenizer: "unicode61",
limit: 10,
@@ -642,7 +628,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db: { prepare } as unknown as Parameters<typeof searchVector>[0]["db"],
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec: [1, 0],
limit: 2,
@@ -667,14 +653,12 @@ describe("searchVector sqlite-vec KNN", () => {
try {
ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: false,
});
const insertChunk = db.prepare(
"INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
);
// Just over 3x the yield batch (FALLBACK_VECTOR_BATCH_SIZE=256), so we
// expect at least 3 yield points to fire during the scan.
@@ -708,7 +692,7 @@ describe("searchVector sqlite-vec KNN", () => {
try {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "yield-model",
queryVec: [1, 0],
limit: 4,
@@ -736,9 +720,7 @@ describe("searchVector sqlite-vec KNN", () => {
const db = new DatabaseSync(":memory:");
ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: false,
});
return db;
@@ -753,7 +735,7 @@ describe("searchVector sqlite-vec KNN", () => {
},
): void {
db.prepare(
"INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
).run(
params.id,
`memory/${params.id}.md`,
@@ -775,7 +757,7 @@ describe("searchVector sqlite-vec KNN", () => {
insertFallbackChunk(db, { id: "other-only", model: "other-model", vector: [1, 0] });
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec: [1, 0],
limit: 5,
@@ -799,7 +781,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "canonical-model",
providerModelAliases: ["/cache/default.gguf"],
queryVec: [1, 0],
@@ -824,7 +806,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "",
queryVec: [1, 0],
limit: 5,
@@ -846,7 +828,7 @@ describe("searchVector sqlite-vec KNN", () => {
insertFallbackChunk(db, { id: "lone", model: "target-model", vector: [1, 0] });
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec: [1, 0],
limit: 5,
@@ -878,7 +860,7 @@ describe("searchVector sqlite-vec KNN", () => {
}
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec: [1, 0],
limit: 3,
@@ -941,7 +923,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec,
limit,
@@ -999,7 +981,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
queryVec: [1, 0],
limit: 2,
@@ -1026,22 +1008,22 @@ describe("searchVector sqlite-vec KNN", () => {
expect(loaded.ok, loaded.error).toBe(true);
ensureMemoryIndexSchema({
db,
embeddingCacheTable: "embedding_cache",
cacheEnabled: false,
ftsTable: "chunks_fts",
ftsEnabled: false,
});
db.exec(`
CREATE VIRTUAL TABLE chunks_vec USING vec0(
CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[2]
);
`);
const insertChunk = db.prepare(
"INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
);
const insertVector = db.prepare(
"INSERT INTO memory_index_chunks_vec (id, embedding) VALUES (?, ?)",
);
const insertVector = db.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)");
const addChunk = (params: { id: string; model: string; vector: [number, number] }) => {
insertChunk.run(
params.id,
@@ -1067,7 +1049,7 @@ describe("searchVector sqlite-vec KNN", () => {
const results = await searchVector({
db,
vectorTable: "chunks_vec",
vectorTable: "memory_index_chunks_vec",
providerModel: "target-model",
providerModelAliases: ["alias-model"],
queryVec: [1, 0],

10
extensions/memory-core/src/memory/manager-search.ts
View File

@@ -159,7 +159,7 @@ export async function searchVector(params: {
// which runs in ~O(log N + k) via the vec0 index, instead of the previous
// full-table scan over vec_distance_cosine(). Keep vec_distance_cosine() in
// the SELECT so `score = 1 - dist` stays in the cosine [0, 1] range the
// downstream merge/minScore pipeline expects. (chunks_vec is created with
// downstream merge/minScore pipeline expects. (memory_index_chunks_vec is created with
// sqlite-vec's default L2 distance, so v.distance cannot be used directly
// for scoring.)
const qBlob = vectorToBlob(params.queryVec);
@@ -170,7 +170,7 @@ export async function searchVector(params: {
` c.source,\n` +
` vec_distance_cosine(v.embedding, ?) AS dist\n` +
` FROM ${params.vectorTable} v\n` +
` JOIN chunks c ON c.id = v.id\n` +
` JOIN memory_index_chunks c ON c.id = v.id\n` +
` WHERE v.embedding MATCH ? AND k = ? AND ${vectorModelFilter}${params.sourceFilterVec.sql}\n` +
` ORDER BY dist ASC\n` +
` LIMIT ?`,
@@ -198,7 +198,7 @@ export async function searchVector(params: {
const matchingChunkCount = readCount(
params.db
.prepare(
`SELECT COUNT(*) AS count FROM chunks c WHERE ${vectorModelFilter}${params.sourceFilterVec.sql}`,
`SELECT COUNT(*) AS count FROM memory_index_chunks c WHERE ${vectorModelFilter}${params.sourceFilterVec.sql}`,
)
.get(...providerModels, ...params.sourceFilterVec.params) as
| { count?: number | bigint }
@@ -257,7 +257,7 @@ async function searchChunksByEmbedding(params: {
// below. The rowid cursor keeps memory bounded without OFFSET rescans.
const stmt = params.db.prepare(
`SELECT rowid, id, path, start_line, end_line, text, embedding, source\n` +
` FROM chunks\n` +
` FROM memory_index_chunks\n` +
` WHERE ${modelFilter} AND rowid > ?${params.sourceFilter.sql}\n` +
` ORDER BY rowid ASC\n` +
` LIMIT ?`,
@@ -348,7 +348,7 @@ export async function searchKeyword(params: {
// Lexical FTS is model-agnostic (issue #48300), but old databases may
// already contain orphaned FTS rows from prior model-scoped cleanup.
const liveChunkClause = ` AND EXISTS (SELECT 1 FROM chunks c WHERE c.id = ${params.ftsTable}.id)`;
const liveChunkClause = ` AND EXISTS (SELECT 1 FROM memory_index_chunks c WHERE c.id = ${params.ftsTable}.id)`;
const substringClause = plan.substringTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
const substringParams = plan.substringTerms.map((term) => `%${escapeLikePattern(term)}%`);

4
extensions/memory-core/src/memory/manager-source-state.ts
View File

@@ -16,8 +16,8 @@ type MemorySourceStateDb = {
};
};
export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash, mtime, size FROM files WHERE source = ?`;
export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM files WHERE path = ? AND source = ?`;
export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash, mtime, size FROM memory_index_sources WHERE source = ?`;
export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM memory_index_sources WHERE path = ? AND source = ?`;
export function loadMemorySourceFileState(params: {
db: MemorySourceStateDb;

4
extensions/memory-core/src/memory/manager-status-state.ts
View File

@@ -20,9 +20,9 @@ type StatusAggregateDb = {
};
export const MEMORY_STATUS_AGGREGATE_SQL =
`SELECT 'files' AS kind, source, COUNT(*) as c FROM files WHERE 1=1__FILTER__ GROUP BY source\n` +
`SELECT 'files' AS kind, source, COUNT(*) as c FROM memory_index_sources WHERE 1=1__FILTER__ GROUP BY source\n` +
`UNION ALL\n` +
`SELECT 'chunks' AS kind, source, COUNT(*) as c FROM chunks WHERE 1=1__FILTER__ GROUP BY source`;
`SELECT 'chunks' AS kind, source, COUNT(*) as c FROM memory_index_chunks WHERE 1=1__FILTER__ GROUP BY source`;
export function resolveInitialMemoryDirty(params: {
hasMemorySource: boolean;

35
extensions/memory-core/src/memory/manager-sync-ops.ts
View File

@@ -29,6 +29,9 @@ import {
isFileMissingError,
listMemoryFiles,
loadSqliteVecExtension,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
normalizeExtraMemoryPaths,
retryTransientMemoryRead,
runWithConcurrency,
@@ -141,9 +144,9 @@ type MemoryReindexRetryState = {
};
const META_KEY = "memory_index_meta_v1";
const VECTOR_TABLE = "chunks_vec";
const FTS_TABLE = "chunks_fts";
const EMBEDDING_CACHE_TABLE = "embedding_cache";
const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE;
const FTS_TABLE = MEMORY_INDEX_FTS_TABLE;
const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE;
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
const SESSION_SYNC_YIELD_EVERY = 10;
@@ -477,7 +480,7 @@ export abstract class MemoryManagerSyncOps {
}
protected hasIndexedChunks(): boolean {
const row = this.db.prepare(`SELECT 1 as found FROM chunks LIMIT 1`).get() as
const row = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as
| { found?: number }
| undefined;
return row?.found === 1;
@@ -485,7 +488,7 @@ export abstract class MemoryManagerSyncOps {
protected hasSemanticChunks(): boolean {
const row = this.db
.prepare(`SELECT 1 as found FROM chunks WHERE model != 'fts-only' LIMIT 1`)
.prepare(`SELECT 1 as found FROM memory_index_chunks WHERE model != 'fts-only' LIMIT 1`)
.get() as { found?: number } | undefined;
return row?.found === 1;
}
@@ -760,9 +763,7 @@ export abstract class MemoryManagerSyncOps {
protected ensureSchema() {
const result = ensureMemoryIndexSchema({
db: this.db,
embeddingCacheTable: EMBEDDING_CACHE_TABLE,
cacheEnabled: this.cache.enabled,
ftsTable: FTS_TABLE,
ftsEnabled: this.fts.enabled,
ftsTokenizer: this.settings.store.fts.tokenizer,
});
@@ -1743,15 +1744,15 @@ export abstract class MemoryManagerSyncOps {
deferIndex?: boolean;
}): Promise<MemorySourceSyncPlan> {
const deleteFileByPathAndSource = this.db.prepare(
`DELETE FROM files WHERE path = ? AND source = ?`,
`DELETE FROM memory_index_sources WHERE path = ? AND source = ?`,
);
const deleteChunksByPathAndSource = this.db.prepare(
`DELETE FROM chunks WHERE path = ? AND source = ?`,
`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`,
);
const deleteVectorRowsByPathAndSource =
this.vector.enabled && this.vector.available
? this.db.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
: null;
const deleteFtsRowsByPathAndSource =
@@ -1873,15 +1874,15 @@ export abstract class MemoryManagerSyncOps {
prefixIndexItems?: MemoryIndexWorkItem[];
}): Promise<MemorySourceSyncPlan> {
const deleteFileByPathAndSource = this.db.prepare(
`DELETE FROM files WHERE path = ? AND source = ?`,
`DELETE FROM memory_index_sources WHERE path = ? AND source = ?`,
);
const deleteChunksByPathAndSource = this.db.prepare(
`DELETE FROM chunks WHERE path = ? AND source = ?`,
`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`,
);
const deleteVectorRowsByPathAndSource =
this.vector.enabled && this.vector.available
? this.db.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
: null;
const deleteFtsRowsByPathAndSource =
@@ -2577,9 +2578,9 @@ export abstract class MemoryManagerSyncOps {
}
protected readMeta(): MemoryIndexMeta | null {
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as
| { value: string }
| undefined;
const row = this.db
.prepare(`SELECT value FROM memory_index_meta WHERE key = ?`)
.get(META_KEY) as { value: string } | undefined;
if (!row?.value) {
this.lastMetaSerialized = null;
return null;
@@ -2601,7 +2602,7 @@ export abstract class MemoryManagerSyncOps {
}
this.db
.prepare(
`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`,
`INSERT INTO memory_index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`,
)
.run(META_KEY, value);
this.lastMetaSerialized = value;

4
extensions/memory-core/src/memory/manager-vector-warning.test.ts
View File

@@ -27,7 +27,7 @@ describe("memory vector degradation warnings", () => {
expect(second).toBe(true);
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
"chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.",
"memory_index_chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.",
);
});
@@ -44,7 +44,7 @@ describe("memory vector degradation warnings", () => {
expect(shown).toBe(true);
expect(warn).toHaveBeenCalledWith(
"chunks_vec not updated — semantic vector embeddings unavailable — no vector dimensions resolved. Vector recall degraded. Further duplicate warnings suppressed.",
"memory_index_chunks_vec not updated — semantic vector embeddings unavailable — no vector dimensions resolved. Vector recall degraded. Further duplicate warnings suppressed.",
);
});

2
extensions/memory-core/src/memory/manager-vector-warning.ts
View File

@@ -22,7 +22,7 @@ export function logMemoryVectorDegradedWrite(params: {
return params.warningShown;
}
params.warn(
`chunks_vec not updated — ${formatMemoryVectorDegradedWriteReason(params.loadError)}. Vector recall degraded. Further duplicate warnings suppressed.`,
`memory_index_chunks_vec not updated — ${formatMemoryVectorDegradedWriteReason(params.loadError)}. Vector recall degraded. Further duplicate warnings suppressed.`,
);
return true;
}

2
extensions/memory-core/src/memory/manager-vector-write.ts
View File

@@ -16,7 +16,7 @@ export function replaceMemoryVectorRow(params: {
embedding: number[];
tableName?: string;
}): void {
const tableName = params.tableName ?? "chunks_vec";
const tableName = params.tableName ?? "memory_index_chunks_vec";
try {
params.db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(params.id);
} catch {}

2
extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts
View File

@@ -101,7 +101,7 @@ describe("memory manager FTS-only reindex", () => {
const db = new DatabaseSync(indexPath);
try {
const row = db
.prepare(`SELECT COUNT(*) as c FROM chunks WHERE text LIKE ?`)
.prepare(`SELECT COUNT(*) as c FROM memory_index_chunks WHERE text LIKE ?`)
.get(`%${term}%`) as { c: number } | undefined;
return row?.c ?? 0;
} finally {

10
extensions/memory-core/src/memory/manager.reindex-recovery.test.ts
View File

@@ -179,7 +179,7 @@ describe("memory manager reindex recovery", () => {
const harness = memoryManager as unknown as ReindexHarness;
const publishedRows = harness.db
.prepare("SELECT path, text FROM chunks ORDER BY path, start_line")
.prepare("SELECT path, text FROM memory_index_chunks ORDER BY path, start_line")
.all();
expect(publishedRows.length).toBeGreaterThan(0);
@@ -192,7 +192,9 @@ describe("memory manager reindex recovery", () => {
"late shadow failure",
);
expect(
harness.db.prepare("SELECT path, text FROM chunks ORDER BY path, start_line").all(),
harness.db
.prepare("SELECT path, text FROM memory_index_chunks ORDER BY path, start_line")
.all(),
).toEqual(publishedRows);
});
@@ -245,7 +247,7 @@ describe("memory manager reindex recovery", () => {
const databasePath = resolveOpenClawAgentSqlitePath({ agentId: "main" });
await fs.mkdir(path.dirname(databasePath), { recursive: true });
const db = new DatabaseSync(databasePath);
db.exec("CREATE TABLE chunks (id TEXT PRIMARY KEY)");
db.exec("CREATE TABLE memory_index_chunks (id TEXT PRIMARY KEY)");
db.close();
const { getMemorySearchManager } = await import("./index.js");
@@ -275,7 +277,7 @@ describe("memory manager reindex recovery", () => {
harness.db
.prepare(
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
`INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(

10
extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts
View File

@@ -104,8 +104,8 @@ describe("memory manager self-heal missing identity with FTS-only chunks", () =>
await fs.mkdir(path.dirname(indexPath), { recursive: true });
const db = new DatabaseSync(indexPath);
db.exec(`
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS chunks (
CREATE TABLE IF NOT EXISTS memory_index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS memory_index_chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
@@ -117,16 +117,16 @@ describe("memory manager self-heal missing identity with FTS-only chunks", () =>
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS files (
CREATE TABLE IF NOT EXISTS memory_index_sources (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
VALUES ('chunk-1', 'MEMORY.md', 'memory', 1, 3, 'hash-1', '${model}', 'Alpha topic keep note', '[]', ${Date.now()});
INSERT INTO files (path, source, hash, mtime, size)
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('MEMORY.md', 'memory', 'hash-1', ${Date.now()}, 100);
`);
db.close();

11
extensions/memory-core/src/memory/manager.ts
View File

@@ -14,6 +14,9 @@ import {
import { extractKeywords } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
import {
readMemoryFile,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
type MemoryEmbeddingProbeResult,
type MemoryProviderStatus,
type MemorySearchManager,
@@ -66,9 +69,9 @@ import {
} from "./manager-sync-control.js";
import { applyTemporalDecayToHybridResults } from "./temporal-decay.js";
const SNIPPET_MAX_CHARS = 700;
const VECTOR_TABLE = "chunks_vec";
const FTS_TABLE = "chunks_fts";
const EMBEDDING_CACHE_TABLE = "embedding_cache";
const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE;
const FTS_TABLE = MEMORY_INDEX_FTS_TABLE;
const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE;
const MEMORY_INDEX_MANAGER_CACHE_KEY = Symbol.for("openclaw.memoryIndexManagerCache");
export const EMBEDDING_PROBE_CACHE_TTL_MS = 30_000;
const KEYWORD_FALLBACK_SEARCH_TERM_LIMIT = 6;
@@ -821,7 +824,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
}
private hasIndexedContent(): boolean {
const chunkRow = this.db.prepare(`SELECT 1 as found FROM chunks LIMIT 1`).get() as
const chunkRow = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as
| {
found?: number;
}

10
extensions/memory-core/src/memory/manager.vector-dedupe.test.ts
View File

@@ -13,7 +13,7 @@ describe("memory vector dedupe", () => {
it("deletes existing vector rows before inserting replacements", () => {
db = new DatabaseSync(":memory:");
db.exec("CREATE TABLE chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
db.exec("CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
replaceMemoryVectorRow({
db,
@@ -23,8 +23,8 @@ describe("memory vector dedupe", () => {
db.exec(`
CREATE TRIGGER fail_if_vector_row_not_deleted
BEFORE INSERT ON chunks_vec
WHEN EXISTS (SELECT 1 FROM chunks_vec WHERE id = NEW.id)
BEFORE INSERT ON memory_index_chunks_vec
WHEN EXISTS (SELECT 1 FROM memory_index_chunks_vec WHERE id = NEW.id)
BEGIN
SELECT RAISE(FAIL, 'vector row not deleted before insert');
END;
@@ -39,7 +39,9 @@ describe("memory vector dedupe", () => {
).toBeUndefined();
const row = db
.prepare("SELECT COUNT(*) as c, length(embedding) as bytes FROM chunks_vec WHERE id = ?")
.prepare(
"SELECT COUNT(*) as c, length(embedding) as bytes FROM memory_index_chunks_vec WHERE id = ?",
)
.get("chunk-1") as { c: number; bytes: number } | undefined;
expect(row?.c).toBe(1);
expect(row?.bytes).toBe(12);

11
packages/memory-host-sdk/src/engine-storage.ts
View File

@@ -39,7 +39,16 @@ export type {
MemorySource,
MemorySyncProgressUpdate,
} from "./host/types.js";
export { ensureMemoryIndexSchema } from "./host/memory-schema.js";
export {
ensureMemoryIndexSchema,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_META_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_INDEX_STATE_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
} from "./host/memory-schema.js";
export { loadSqliteVecExtension } from "./host/sqlite-vec.js";
export {
closeMemorySqliteWalMaintenance,

97
packages/memory-host-sdk/src/host/memory-schema.ts
View File

@@ -4,32 +4,34 @@ import { formatErrorMessage } from "./error-utils.js";
// SQLite schema setup for builtin memory index, embedding cache, and FTS.
/** Ensure memory index tables and optional FTS/cache tables exist. */
export const MEMORY_INDEX_META_TABLE = "memory_index_meta";
export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources";
export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
export const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache";
export const MEMORY_INDEX_STATE_TABLE = "memory_index_state";
export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec";
/** Ensure canonical memory index tables and the optional FTS table exist. */
export function ensureMemoryIndexSchema(params: {
db: DatabaseSync;
embeddingCacheTable: string;
cacheEnabled: boolean;
ftsTable: string;
ftsEnabled: boolean;
ftsTokenizer?: "unicode61" | "trigram";
}): { ftsAvailable: boolean; ftsError?: string } {
params.db.exec(`
CREATE TABLE IF NOT EXISTS meta (
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS files (
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS chunks (
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
@@ -41,49 +43,52 @@ export function ensureMemoryIndexSchema(params: {
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS memory_index_state (
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} (
id INTEGER PRIMARY KEY CHECK (id = 1),
revision INTEGER NOT NULL
);
INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0);
INSERT OR IGNORE INTO ${MEMORY_INDEX_STATE_TABLE} (id, revision) VALUES (1, 0);
CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_insert
AFTER INSERT ON files
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_update
AFTER UPDATE ON files
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update
AFTER UPDATE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_files_revision_after_delete
AFTER DELETE ON files
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_insert
AFTER INSERT ON chunks
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert
AFTER INSERT ON ${MEMORY_INDEX_CHUNKS_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_update
AFTER UPDATE ON chunks
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update
AFTER UPDATE ON ${MEMORY_INDEX_CHUNKS_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_chunks_revision_after_delete
AFTER DELETE ON chunks
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete
AFTER DELETE ON ${MEMORY_INDEX_CHUNKS_TABLE}
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path
ON ${MEMORY_INDEX_CHUNKS_TABLE}(path);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
ON ${MEMORY_INDEX_CHUNKS_TABLE}(source);
`);
if (params.cacheEnabled) {
params.db.exec(`
CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
CREATE TABLE IF NOT EXISTS ${MEMORY_EMBEDDING_CACHE_TABLE} (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
@@ -93,10 +98,9 @@ export function ensureMemoryIndexSchema(params: {
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
);
CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at
ON ${MEMORY_EMBEDDING_CACHE_TABLE}(updated_at);
`);
params.db.exec(
`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`,
);
}
let ftsAvailable = false;
@@ -106,7 +110,7 @@ export function ensureMemoryIndexSchema(params: {
const tokenizer = params.ftsTokenizer ?? "unicode61";
const tokenizeClause = tokenizer === "trigram" ? `, tokenize='trigram case_sensitive 0'` : "";
params.db.exec(
`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n` +
`CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_FTS_TABLE} USING fts5(\n` +
` text,\n` +
` id UNINDEXED,\n` +
` path UNINDEXED,\n` +
@@ -124,24 +128,5 @@ export function ensureMemoryIndexSchema(params: {
}
}
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
return { ftsAvailable, ...(ftsError ? { ftsError } : {}) };
}
/** Add a missing shipped column without rebuilding existing memory tables. */
function ensureColumn(
db: DatabaseSync,
table: "files" | "chunks",
column: string,
definition: string,
): void {
const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
if (rows.some((row) => row.name === column)) {
return;
}
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
}

26
src/commands/status.scan.shared.ts
View File

@@ -18,7 +18,12 @@ import { normalizeControlUiBasePath } from "../gateway/control-ui-shared.js";
import { resolveGatewayProbeTarget } from "../gateway/probe-target.js";
import type { GatewayProbeResult, probeGateway as probeGatewayFn } from "../gateway/probe.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import type { MemoryProviderStatus } from "../memory-host-sdk/engine-storage.js";
import {
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_META_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
type MemoryProviderStatus,
} from "../memory-host-sdk/engine-storage.js";
import { defaultSlotIdForKey } from "../plugins/slots.js";
import { createLazyImportLoader } from "../shared/lazy-promise.js";
import { resolveTailscalePublishedHost } from "../shared/tailscale-status.js";
@@ -51,24 +56,29 @@ function hasBuiltInMemoryState(databasePath: string): boolean {
let db: DatabaseSync | undefined;
try {
db = new DatabaseSync(databasePath, { readOnly: true });
const builtInMemoryTables = [
MEMORY_INDEX_META_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_INDEX_CHUNKS_TABLE,
] as const;
const tableNames = new Set(
(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks')",
)
.all() as Array<{ name?: unknown }>
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)`)
.all(...builtInMemoryTables) as Array<{ name?: unknown }>
)
.map((row) => row.name)
.filter((name): name is string => typeof name === "string"),
);
if (
tableNames.has("meta") &&
db.prepare("SELECT 1 AS ok FROM meta WHERE key = ? LIMIT 1").get(MEMORY_INDEX_META_KEY)
tableNames.has(MEMORY_INDEX_META_TABLE) &&
db
.prepare(`SELECT 1 AS ok FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ? LIMIT 1`)
.get(MEMORY_INDEX_META_KEY)
) {
return true;
}
for (const tableName of ["files", "chunks"] as const) {
for (const tableName of [MEMORY_INDEX_SOURCES_TABLE, MEMORY_INDEX_CHUNKS_TABLE] as const) {
if (
tableNames.has(tableName) &&
db.prepare(`SELECT 1 AS ok FROM ${tableName} LIMIT 1`).get()

3
src/memory-host-sdk/engine-storage.ts
View File

@@ -3,6 +3,9 @@
* path stable while the shared SDK package owns provider status semantics.
*/
export {
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_META_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
resolveMemoryBackendConfig,
type MemoryProviderStatus,
} from "../../packages/memory-host-sdk/src/engine-storage.js";

7
src/plugin-sdk/memory-core-host-engine-storage.ts
View File

@@ -19,6 +19,13 @@ export {
isTransientMemoryReadError,
listMemoryFiles,
loadSqliteVecExtension,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_META_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_INDEX_STATE_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
normalizeExtraMemoryPaths,
parseEmbedding,
readMemoryFile,

46
src/state/openclaw-agent-db.generated.d.ts vendored
View File

@@ -31,6 +31,47 @@ export interface CacheEntries {
value_json: string | null;
}
export interface MemoryEmbeddingCache {
dims: number | null;
embedding: string;
hash: string;
model: string;
provider: string;
provider_key: string;
updated_at: number;
}
export interface MemoryIndexChunks {
embedding: string;
end_line: number;
hash: string;
id: string | null;
model: string;
path: string;
source: Generated<string>;
start_line: number;
text: string;
updated_at: number;
}
export interface MemoryIndexMeta {
key: string | null;
value: string;
}
export interface MemoryIndexSources {
hash: string;
mtime: number;
path: string | null;
size: number;
source: Generated<string>;
}
export interface MemoryIndexState {
id: Generated<number>;
revision: number;
}
export interface SchemaMeta {
agent_id: string | null;
app_version: string | null;
@@ -45,5 +86,10 @@ export interface DB {
auth_profile_state: AuthProfileState;
auth_profile_store: AuthProfileStore;
cache_entries: CacheEntries;
memory_embedding_cache: MemoryEmbeddingCache;
memory_index_chunks: MemoryIndexChunks;
memory_index_meta: MemoryIndexMeta;
memory_index_sources: MemoryIndexSources;
memory_index_state: MemoryIndexState;
schema_meta: SchemaMeta;
}

91
src/state/openclaw-agent-schema.generated.ts
View File

@@ -40,4 +40,93 @@ CREATE TABLE IF NOT EXISTS auth_profile_state (
state_key TEXT NOT NULL PRIMARY KEY,
state_json TEXT NOT NULL,
updated_at INTEGER NOT NULL
);\n`;
);
CREATE TABLE IF NOT EXISTS memory_index_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_index_sources (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_index_chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_embedding_cache (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding TEXT NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
);
CREATE TABLE IF NOT EXISTS memory_index_state (
id INTEGER PRIMARY KEY CHECK (id = 1),
revision INTEGER NOT NULL
);
INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0);
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert
AFTER INSERT ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update
AFTER UPDATE ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete
AFTER DELETE ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert
AFTER INSERT ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update
AFTER UPDATE ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete
AFTER DELETE ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at
ON memory_embedding_cache(updated_at);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path
ON memory_index_chunks(path);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
ON memory_index_chunks(source);\n`;

89
src/state/openclaw-agent-schema.sql
View File

@@ -36,3 +36,92 @@ CREATE TABLE IF NOT EXISTS auth_profile_state (
state_json TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_index_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_index_sources (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_index_chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_embedding_cache (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding TEXT NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
);
CREATE TABLE IF NOT EXISTS memory_index_state (
id INTEGER PRIMARY KEY CHECK (id = 1),
revision INTEGER NOT NULL
);
INSERT OR IGNORE INTO memory_index_state (id, revision) VALUES (1, 0);
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_insert
AFTER INSERT ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_update
AFTER UPDATE ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_sources_revision_after_delete
AFTER DELETE ON memory_index_sources
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_insert
AFTER INSERT ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_update
AFTER UPDATE ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_delete
AFTER DELETE ON memory_index_chunks
BEGIN
UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1;
END;
CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at
ON memory_embedding_cache(updated_at);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path
ON memory_index_chunks(path);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
ON memory_index_chunks(source);