From d64db0fc3d1c8ddec386dc5901603fe102c5a572 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 16 Jun 2026 13:08:33 +0200 Subject: [PATCH] fix(memory): load vector extension before publish --- .../memory-core/src/memory/manager-db.test.ts | 55 +++++++++++++++++-- .../memory-core/src/memory/manager-db.ts | 32 ++++++++++- .../src/memory/manager-sync-ops.ts | 3 +- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/extensions/memory-core/src/memory/manager-db.test.ts b/extensions/memory-core/src/memory/manager-db.test.ts index 81c8175577ef..4caaa1f8b5d1 100644 --- a/extensions/memory-core/src/memory/manager-db.test.ts +++ b/extensions/memory-core/src/memory/manager-db.test.ts @@ -3,7 +3,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; -import { ensureMemoryIndexSchema } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { + ensureMemoryIndexSchema, + loadSqliteVecExtension, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { cleanupAgedMemoryReindexTempFiles, @@ -37,7 +40,7 @@ describe("memory manager database publication", () => { await fs.rm(fixtureRoot, { recursive: true, force: true }); }); - it("removes a stale vector table when the shadow index has no vectors", () => { + it("removes a stale vector table when the shadow index has no vectors", async () => { const targetPath = path.join(fixtureRoot, "target.sqlite"); const sourcePath = path.join(fixtureRoot, "source.sqlite"); const targetDb = new DatabaseSync(targetPath); @@ -49,7 +52,7 @@ describe("memory manager database publication", () => { targetDb.prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)").run("stale", "[]"); sourceDb.close(); - publishMemoryDatabaseTables({ + await publishMemoryDatabaseTables({ targetDb, sourcePath, metaKey: "memory_index_meta", @@ -69,7 +72,47 @@ describe("memory manager database publication", () => { } }); - it("rejects a stale shadow publish after a concurrent live memory update", () => { + it("loads sqlite-vec on the target before publishing a shadow vector table", async () => { + const targetPath = path.join(fixtureRoot, "target.sqlite"); + const sourcePath = path.join(fixtureRoot, "source.sqlite"); + const targetDb = new DatabaseSync(targetPath, { allowExtension: true }); + const sourceDb = new DatabaseSync(sourcePath, { allowExtension: true }); + try { + ensureTestMemorySchema(targetDb); + ensureTestMemorySchema(sourceDb); + const sourceVector = await loadSqliteVecExtension({ db: sourceDb }); + if (!sourceVector.ok) { + return; + } + sourceDb.exec(` + CREATE VIRTUAL TABLE chunks_vec USING vec0( + id TEXT PRIMARY KEY, + embedding FLOAT[3] + ) + `); + sourceDb + .prepare("INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)") + .run("vector", JSON.stringify([0, 1, 0])); + sourceDb.close(); + + await publishMemoryDatabaseTables({ + targetDb, + sourcePath, + metaKey: "memory_index_meta", + expectedRevision: readMemoryDatabaseRevision(targetDb), + vectorExtensionPath: sourceVector.extensionPath, + }); + + expect(targetDb.prepare("SELECT id FROM chunks_vec").all()).toEqual([{ id: "vector" }]); + } finally { + try { + sourceDb.close(); + } catch {} + targetDb.close(); + } + }); + + it("rejects a stale shadow publish after a concurrent live memory update", async () => { const targetPath = path.join(fixtureRoot, "target.sqlite"); const sourcePath = path.join(fixtureRoot, "source.sqlite"); const targetDb = new DatabaseSync(targetPath); @@ -92,14 +135,14 @@ describe("memory manager database publication", () => { concurrentDb.close(); concurrentDb = undefined; - expect(() => + await expect( publishMemoryDatabaseTables({ targetDb, sourcePath, metaKey: "memory_index_meta", expectedRevision, }), - ).toThrow(/changed while full reindex was building/); + ).rejects.toThrow(/changed while full reindex was building/); expect( targetDb.prepare("SELECT hash FROM files WHERE path = ?").get("memory.md"), ).toEqual({ hash: "newer" }); diff --git a/extensions/memory-core/src/memory/manager-db.ts b/extensions/memory-core/src/memory/manager-db.ts index f866f5192cdd..5741c2a93bd6 100644 --- a/extensions/memory-core/src/memory/manager-db.ts +++ b/extensions/memory-core/src/memory/manager-db.ts @@ -6,6 +6,7 @@ import { closeMemorySqliteWalMaintenance, configureMemorySqliteWalMaintenance, ensureDir, + loadSqliteVecExtension, requireNodeSqlite, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { @@ -67,6 +68,17 @@ function readTableSql(db: DatabaseSync, schema: string, tableName: string): stri return typeof row?.sql === "string" && row.sql.trim() ? row.sql : null; } +function hasSqliteVecExtension(db: DatabaseSync): boolean { + try { + const row = db.prepare("SELECT vec_version() AS version").get() as + | { version?: unknown } + | undefined; + return typeof row?.version === "string" && row.version.trim().length > 0; + } catch { + return false; + } +} + export function readMemoryDatabaseRevision(db: DatabaseSync): number { const row = db .prepare("SELECT revision FROM memory_index_state WHERE id = ?") @@ -104,14 +116,30 @@ function replaceVirtualTable(params: { } /** Publish a completed shadow memory index without replacing the shared agent database file. */ -export function publishMemoryDatabaseTables(params: { +export async function publishMemoryDatabaseTables(params: { targetDb: DatabaseSync; sourcePath: string; metaKey: string; expectedRevision: number; -}): void { + vectorExtensionPath?: string; +}): Promise { params.targetDb.prepare(`ATTACH DATABASE ? AS ${MEMORY_REINDEX_SCHEMA}`).run(params.sourcePath); try { + if ( + tableExists(params.targetDb, MEMORY_REINDEX_SCHEMA, "chunks_vec") && + !hasSqliteVecExtension(params.targetDb) + ) { + const loaded = await loadSqliteVecExtension({ + db: params.targetDb, + extensionPath: params.vectorExtensionPath, + }); + if (!loaded.ok) { + throw new Error( + `Failed to load sqlite-vec before publishing the full memory reindex: ` + + `${loaded.error ?? "unknown sqlite-vec load error"}`, + ); + } + } runSqliteImmediateTransactionSync(params.targetDb, () => { const liveRevision = readMemoryDatabaseRevision(params.targetDb); if (liveRevision !== params.expectedRevision) { diff --git a/extensions/memory-core/src/memory/manager-sync-ops.ts b/extensions/memory-core/src/memory/manager-sync-ops.ts index 3f5271547848..69d253b8f78f 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.ts @@ -2530,11 +2530,12 @@ export abstract class MemoryManagerSyncOps { closeMemoryDatabase(tempDb); tempDbClosed = true; - publishMemoryDatabaseTables({ + await publishMemoryDatabaseTables({ targetDb: originalDb, sourcePath: tempDbPath, metaKey: META_KEY, expectedRevision: originalRevision, + vectorExtensionPath: this.vector.extensionPath, }); this.db = originalDb;