From d9ffbb3ed6c0237a6c6235079f9471bcb6c12811 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 6 Aug 2026 15:57:49 -0700 Subject: [PATCH] fix(memory): report persisted vector index state on unprobed status path (#120048) --- docs/cli/memory.md | 14 +++--- .../memory-core/src/cli-status.runtime.ts | 11 ++++- extensions/memory-core/src/cli.test.ts | 35 +++++++++++++++ .../memory-core/src/memory/index.test.ts | 38 ++++++++++++++++ .../manager-vector-rebuild-state.test.ts | 43 +++++++++++++++++++ .../memory/manager-vector-rebuild-state.ts | 31 +++++++++++-- extensions/memory-core/src/memory/manager.ts | 7 +++ .../memory-host-sdk/src/engine-storage.ts | 1 + packages/memory-host-sdk/src/host/types.ts | 7 +++ .../memory-core-host-engine-storage.ts | 1 + 10 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 extensions/memory-core/src/memory/manager-vector-rebuild-state.test.ts diff --git a/docs/cli/memory.md b/docs/cli/memory.md index a5ba47b378aa..e0b1b2171398 100644 --- a/docs/cli/memory.md +++ b/docs/cli/memory.md @@ -27,13 +27,13 @@ openclaw memory status [--agent ] [--deep] [--index] [--fix] [--json] [--ver Without `--agent`, runs for every agent in `agents.entries`; if no agent list is configured, falls back to the default agent. -| Flag | Effect | -| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--deep` | Probe vector-store, embedding-provider, and semantic-search readiness (implies extra provider calls). Plain `memory status` stays fast and skips this; unknown vector/semantic state means it was not probed. QMD lexical `searchMode: "search"` always skips semantic vector probes, even with `--deep`. | -| `--index` | Reindex if the store is dirty. Implies `--deep`. | -| `--fix` | Repair stale recall locks and normalize promotion metadata. | -| `--json` | Print JSON. | -| `--verbose` | Emit detailed per-phase logs. | +| Flag | Effect | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--deep` | Probe vector-store, embedding-provider, and semantic-search readiness (implies extra provider calls). Plain `memory status` stays fast and skips this; a complete persisted index is shown as `indexed (unprobed)`, while unknown vector/semantic state means it was not probed. QMD lexical `searchMode: "search"` always skips semantic vector probes, even with `--deep`. | +| `--index` | Reindex if the store is dirty. Implies `--deep`. | +| `--fix` | Repair stale recall locks and normalize promotion metadata. | +| `--json` | Print JSON. | +| `--verbose` | Emit detailed per-phase logs. | If the `Dreaming` line stays `off` even with `dreaming.enabled: true`, or scheduled sweeps never seem to run, the managed dreaming cron depends on the diff --git a/extensions/memory-core/src/cli-status.runtime.ts b/extensions/memory-core/src/cli-status.runtime.ts index 0f3ffeb1248f..8080c3c61064 100644 --- a/extensions/memory-core/src/cli-status.runtime.ts +++ b/extensions/memory-core/src/cli-status.runtime.ts @@ -440,7 +440,16 @@ export async function runMemoryStatus( lines.push(`${label(lineLabel)} ${vectorColor(state)}`); }; if (status.backend === "builtin") { - const storeState = formatVectorState(status.vector.storeAvailable); + const storeState = + status.vector.storeAvailable === undefined && status.vector.enabled + ? status.vector.index?.state === "complete" + ? "indexed (unprobed)" + : status.vector.index?.state === "incomplete" + ? "index incomplete (unprobed)" + : status.vector.index?.state === "unverified" + ? "index unverified (unprobed)" + : formatVectorState(undefined) + : formatVectorState(status.vector.storeAvailable); formatVectorLine("Vector store", storeState); if (status.vector.semanticAvailable !== undefined) { formatVectorLine("Semantic vectors", formatVectorState(status.vector.semanticAvailable)); diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index 573fef5e9453..f116b43ae1c2 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -612,6 +612,40 @@ describe("memory cli", () => { expect(close).toHaveBeenCalled(); }); + it("reports a complete persisted vector index without probing the store", async () => { + const close = vi.fn(async () => {}); + const probeVectorStoreAvailability = vi.fn(async () => { + throw new Error("unexpected vector store probe"); + }); + const probeVectorAvailability = vi.fn(async () => { + throw new Error("unexpected vector probe"); + }); + const probeEmbeddingAvailability = vi.fn(async () => { + throw new Error("unexpected embedding probe"); + }); + mockManager({ + probeVectorStoreAvailability, + probeVectorAvailability, + probeEmbeddingAvailability, + status: () => + makeMemoryStatus({ + chunks: 5, + vector: { enabled: true, index: { state: "complete" } }, + }), + close, + }); + + const log = spyRuntimeLogs(defaultRuntime); + await runMemoryCli(["status"]); + + expect(probeVectorStoreAvailability).not.toHaveBeenCalled(); + expect(probeVectorAvailability).not.toHaveBeenCalled(); + expect(probeEmbeddingAvailability).not.toHaveBeenCalled(); + expectLogged(log, "Vector store: indexed (unprobed)"); + expectNotLogged(log, "Vector store: unknown"); + expect(close).toHaveBeenCalled(); + }); + it("fans JSON status out to every keyed agent entry", async () => { const agentIds = ["main", ...Array.from({ length: 21 }, (_, index) => `agent-${index + 1}`)]; getRuntimeConfig.mockReturnValue({ @@ -772,6 +806,7 @@ describe("memory cli", () => { dirty: true, vector: { enabled: true, + index: { state: "complete" }, storeAvailable: false, semanticAvailable: false, available: false, diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index f8e10b51dfc2..3aa55f86c632 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -3090,6 +3090,44 @@ describe("memory index", () => { expect(status.vector?.available).toBeUndefined(); }); + it("reports persisted vector index state on the unprobed status path", async () => { + const cfg = createCfg({ provider: "gemini", vectorEnabled: true }); + const emptyManager = await getFreshManager(cfg, "status"); + try { + const emptyStatus = emptyManager.status(); + expect(emptyStatus.chunks).toBe(0); + expect(emptyStatus.vector?.storeAvailable).toBeUndefined(); + expect(emptyStatus.vector?.index).toEqual({ state: "empty" }); + } finally { + await emptyManager.close?.(); + } + + const indexingManager = await getFreshManager(cfg); + try { + await indexingManager.sync({ reason: "test", force: true }); + expect(indexingManager.status().chunks).toBeGreaterThan(0); + } finally { + await indexingManager.close?.(); + } + + const statusManager = await getFreshManager(cfg, "status"); + try { + expect(Reflect.get(statusManager, "vector")).toMatchObject({ available: null, dims: 4 }); + expect(statusManager.status().vector).toMatchObject({ + index: { state: "complete" }, + storeAvailable: undefined, + }); + + const db = Reflect.get(statusManager, "db") as DatabaseSync; + db.prepare("UPDATE memory_index_meta SET value = '1' WHERE key = ?").run( + "memory_vector_rebuild_v1", + ); + expect(statusManager.status().vector?.index).toEqual({ state: "incomplete" }); + } finally { + await statusManager.close?.(); + } + }); + it("keeps current vector indexes clean after vector store probing", async () => { const cfg = createCfg({ provider: "gemini" }); const manager = await getFreshManager(cfg); diff --git a/extensions/memory-core/src/memory/manager-vector-rebuild-state.test.ts b/extensions/memory-core/src/memory/manager-vector-rebuild-state.test.ts new file mode 100644 index 000000000000..dfe9d66c687f --- /dev/null +++ b/extensions/memory-core/src/memory/manager-vector-rebuild-state.test.ts @@ -0,0 +1,43 @@ +import { DatabaseSync } from "node:sqlite"; +import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { resolvePersistedMemoryVectorIndexState } from "./manager-vector-rebuild-state.js"; + +describe("persisted memory vector index state", () => { + let db: DatabaseSync; + + beforeEach(() => { + db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE ${MEMORY_INDEX_META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY); + `); + }); + + afterEach(() => db.close()); + + it("trusts a clean published index even when an incremental first write has no dimensions metadata", () => { + db.prepare(`INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, 'clean')`).run( + "memory_vector_rebuild_v1", + ); + + expect( + resolvePersistedMemoryVectorIndexState({ + db, + vectorTable: "memory_index_chunks_vec", + hasSemanticChunks: true, + }), + ).toEqual({ state: "complete" }); + }); + + it("keeps a pre-marker vector index unverified", () => { + expect( + resolvePersistedMemoryVectorIndexState({ + db, + vectorTable: "memory_index_chunks_vec", + metaVectorDims: 768, + hasSemanticChunks: true, + }), + ).toEqual({ state: "unverified" }); + }); +}); diff --git a/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts b/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts index abea46596f14..4d18eed13453 100644 --- a/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts +++ b/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts @@ -1,6 +1,9 @@ // Memory Core plugin module owns persisted vector completeness state. import type { DatabaseSync } from "node:sqlite"; -import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { + MEMORY_INDEX_META_TABLE, + type MemoryVectorIndexState, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; const VECTOR_REBUILD_META_KEY = "memory_vector_rebuild_v1"; @@ -30,16 +33,36 @@ export function requiresMemoryVectorRebuild(params: { metaVectorDims?: number; hasSemanticChunks: boolean; }): boolean { + const state = resolvePersistedMemoryVectorIndexState(params).state; + return state === "incomplete" || state === "unverified"; +} + +export function resolvePersistedMemoryVectorIndexState(params: { + db: DatabaseSync; + vectorTable: string; + metaVectorDims?: number; + hasSemanticChunks: boolean; +}): MemoryVectorIndexState { const row = params.db .prepare(`SELECT value FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ?`) .get(VECTOR_REBUILD_META_KEY) as { value?: unknown } | undefined; if (row?.value === "1") { - return true; + return { state: "incomplete" }; } if (!vectorTableExists(params.db, params.vectorTable)) { - return Boolean(params.metaVectorDims && params.hasSemanticChunks); + return params.metaVectorDims && params.hasSemanticChunks + ? { state: "incomplete" } + : { state: "empty" }; + } + // The clean marker is published with the vector table. A later first + // incremental write can populate that table without rewriting vectorDims. + if (row?.value === "clean") { + return params.hasSemanticChunks ? { state: "complete" } : { state: "empty" }; + } + if (params.hasSemanticChunks && !params.metaVectorDims) { + return { state: "incomplete" }; } // Existing releases had no completeness marker. Rebuild their vector table // once rather than assuming it has neither missing nor orphaned rows. - return row?.value !== "clean"; + return { state: "unverified" }; } diff --git a/extensions/memory-core/src/memory/manager.ts b/extensions/memory-core/src/memory/manager.ts index 3e32e941af3a..4287bdc7c6d1 100644 --- a/extensions/memory-core/src/memory/manager.ts +++ b/extensions/memory-core/src/memory/manager.ts @@ -92,6 +92,7 @@ import { runMemorySyncWithReadonlyRecovery, type MemoryReadonlyRecoveryState, } from "./manager-sync-control.js"; +import { resolvePersistedMemoryVectorIndexState } from "./manager-vector-rebuild-state.js"; import { applyProjectRanking } from "./project-ranking.js"; import { applyTemporalDecayToHybridResults } from "./temporal-decay.js"; @@ -2194,6 +2195,12 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem : undefined, vector: { enabled: this.vector.enabled, + index: resolvePersistedMemoryVectorIndexState({ + db: this.db, + vectorTable: VECTOR_TABLE, + metaVectorDims: this.vector.dims, + hasSemanticChunks: this.hasSemanticChunks(), + }), storeAvailable: this.vector.available ?? undefined, semanticAvailable: this.vector.semanticAvailable, available: this.vector.semanticAvailable, diff --git a/packages/memory-host-sdk/src/engine-storage.ts b/packages/memory-host-sdk/src/engine-storage.ts index 272cd9dc39ea..489c8fb4c7df 100644 --- a/packages/memory-host-sdk/src/engine-storage.ts +++ b/packages/memory-host-sdk/src/engine-storage.ts @@ -52,6 +52,7 @@ export type { MemorySource, MemorySyncParams, MemorySyncProgressUpdate, + MemoryVectorIndexState, } from "./host/types.js"; export { dropMemoryPathFtsTriggers, diff --git a/packages/memory-host-sdk/src/host/types.ts b/packages/memory-host-sdk/src/host/types.ts index 025ccf23f743..b7fe08ade570 100644 --- a/packages/memory-host-sdk/src/host/types.ts +++ b/packages/memory-host-sdk/src/host/types.ts @@ -122,6 +122,12 @@ export type MemoryReadResult = { }; /** Aggregated memory backend status for CLI/UI diagnostics. */ +export type MemoryVectorIndexState = + | { state: "empty" } + | { state: "complete" } + | { state: "incomplete" } + | { state: "unverified" }; + export type MemoryProviderStatus = { backend: "builtin" | "qmd"; provider: string; @@ -140,6 +146,7 @@ export type MemoryProviderStatus = { fallback?: { from: string; reason?: string }; vector?: { enabled: boolean; + index?: MemoryVectorIndexState; storeAvailable?: boolean; semanticAvailable?: boolean; available?: boolean; diff --git a/src/plugin-sdk/memory-core-host-engine-storage.ts b/src/plugin-sdk/memory-core-host-engine-storage.ts index 3c77ff03e516..b7e518620db0 100644 --- a/src/plugin-sdk/memory-core-host-engine-storage.ts +++ b/src/plugin-sdk/memory-core-host-engine-storage.ts @@ -87,4 +87,5 @@ export type { ResolvedMemoryBackendConfig, ResolvedQmdConfig, ResolvedQmdMcporterConfig, + MemoryVectorIndexState, } from "../../packages/memory-host-sdk/src/engine-storage.js";