dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-09 03:22:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(memory): report persisted vector index state on unprobed status path (#120048)

Browse Source
This commit is contained in:
Peter Steinberger
2026-08-06 15:57:49 -07:00
committed by GitHub
parent d0fe7bcb3c
commit d9ffbb3ed6
10 changed files with 176 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
docs/cli/memory.md
View File

@@ -27,13 +27,13 @@ openclaw memory status [--agent <id>] [--deep] [--index] [--fix] [--json] [--ver
Without `--agent`, runs for every agent in `agents.entries`; if no agent list is
configured, falls back to the default agent.
| Flag | Effect |
| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--deep` | Probe vector-store, embedding-provider, and semantic-search readiness (implies extra provider calls). Plain `memory status` stays fast and skips this; unknown vector/semantic state means it was not probed. QMD lexical `searchMode: "search"` always skips semantic vector probes, even with `--deep`. |
| `--index` | Reindex if the store is dirty. Implies `--deep`. |
| `--fix` | Repair stale recall locks and normalize promotion metadata. |
| `--json` | Print JSON. |
| `--verbose` | Emit detailed per-phase logs. |
| Flag | Effect |
| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--deep` | Probe vector-store, embedding-provider, and semantic-search readiness (implies extra provider calls). Plain `memory status` stays fast and skips this; a complete persisted index is shown as `indexed (unprobed)`, while unknown vector/semantic state means it was not probed. QMD lexical `searchMode: "search"` always skips semantic vector probes, even with `--deep`. |
| `--index` | Reindex if the store is dirty. Implies `--deep`. |
| `--fix` | Repair stale recall locks and normalize promotion metadata. |
| `--json` | Print JSON. |
| `--verbose` | Emit detailed per-phase logs. |
If the `Dreaming` line stays `off` even with `dreaming.enabled: true`, or
scheduled sweeps never seem to run, the managed dreaming cron depends on the

11
extensions/memory-core/src/cli-status.runtime.ts
View File

@@ -440,7 +440,16 @@ export async function runMemoryStatus(
lines.push(`${label(lineLabel)} ${vectorColor(state)}`);
};
if (status.backend === "builtin") {
const storeState = formatVectorState(status.vector.storeAvailable);
const storeState =
status.vector.storeAvailable === undefined && status.vector.enabled
? status.vector.index?.state === "complete"
? "indexed (unprobed)"
: status.vector.index?.state === "incomplete"
? "index incomplete (unprobed)"
: status.vector.index?.state === "unverified"
? "index unverified (unprobed)"
: formatVectorState(undefined)
: formatVectorState(status.vector.storeAvailable);
formatVectorLine("Vector store", storeState);
if (status.vector.semanticAvailable !== undefined) {
formatVectorLine("Semantic vectors", formatVectorState(status.vector.semanticAvailable));

35
extensions/memory-core/src/cli.test.ts
View File

@@ -612,6 +612,40 @@ describe("memory cli", () => {
expect(close).toHaveBeenCalled();
});
it("reports a complete persisted vector index without probing the store", async () => {
const close = vi.fn(async () => {});
const probeVectorStoreAvailability = vi.fn(async () => {
throw new Error("unexpected vector store probe");
});
const probeVectorAvailability = vi.fn(async () => {
throw new Error("unexpected vector probe");
});
const probeEmbeddingAvailability = vi.fn(async () => {
throw new Error("unexpected embedding probe");
});
mockManager({
probeVectorStoreAvailability,
probeVectorAvailability,
probeEmbeddingAvailability,
status: () =>
makeMemoryStatus({
chunks: 5,
vector: { enabled: true, index: { state: "complete" } },
}),
close,
});
const log = spyRuntimeLogs(defaultRuntime);
await runMemoryCli(["status"]);
expect(probeVectorStoreAvailability).not.toHaveBeenCalled();
expect(probeVectorAvailability).not.toHaveBeenCalled();
expect(probeEmbeddingAvailability).not.toHaveBeenCalled();
expectLogged(log, "Vector store: indexed (unprobed)");
expectNotLogged(log, "Vector store: unknown");
expect(close).toHaveBeenCalled();
});
it("fans JSON status out to every keyed agent entry", async () => {
const agentIds = ["main", ...Array.from({ length: 21 }, (_, index) => `agent-${index + 1}`)];
getRuntimeConfig.mockReturnValue({
@@ -772,6 +806,7 @@ describe("memory cli", () => {
dirty: true,
vector: {
enabled: true,
index: { state: "complete" },
storeAvailable: false,
semanticAvailable: false,
available: false,

38
extensions/memory-core/src/memory/index.test.ts
View File

@@ -3090,6 +3090,44 @@ describe("memory index", () => {
expect(status.vector?.available).toBeUndefined();
});
it("reports persisted vector index state on the unprobed status path", async () => {
const cfg = createCfg({ provider: "gemini", vectorEnabled: true });
const emptyManager = await getFreshManager(cfg, "status");
try {
const emptyStatus = emptyManager.status();
expect(emptyStatus.chunks).toBe(0);
expect(emptyStatus.vector?.storeAvailable).toBeUndefined();
expect(emptyStatus.vector?.index).toEqual({ state: "empty" });
} finally {
await emptyManager.close?.();
}
const indexingManager = await getFreshManager(cfg);
try {
await indexingManager.sync({ reason: "test", force: true });
expect(indexingManager.status().chunks).toBeGreaterThan(0);
} finally {
await indexingManager.close?.();
}
const statusManager = await getFreshManager(cfg, "status");
try {
expect(Reflect.get(statusManager, "vector")).toMatchObject({ available: null, dims: 4 });
expect(statusManager.status().vector).toMatchObject({
index: { state: "complete" },
storeAvailable: undefined,
});
const db = Reflect.get(statusManager, "db") as DatabaseSync;
db.prepare("UPDATE memory_index_meta SET value = '1' WHERE key = ?").run(
"memory_vector_rebuild_v1",
);
expect(statusManager.status().vector?.index).toEqual({ state: "incomplete" });
} finally {
await statusManager.close?.();
}
});
it("keeps current vector indexes clean after vector store probing", async () => {
const cfg = createCfg({ provider: "gemini" });
const manager = await getFreshManager(cfg);

43
extensions/memory-core/src/memory/manager-vector-rebuild-state.test.ts Normal file
View File

@@ -0,0 +1,43 @@
import { DatabaseSync } from "node:sqlite";
import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { resolvePersistedMemoryVectorIndexState } from "./manager-vector-rebuild-state.js";
describe("persisted memory vector index state", () => {
let db: DatabaseSync;
beforeEach(() => {
db = new DatabaseSync(":memory:");
db.exec(`
CREATE TABLE ${MEMORY_INDEX_META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE memory_index_chunks_vec (id TEXT PRIMARY KEY);
`);
});
afterEach(() => db.close());
it("trusts a clean published index even when an incremental first write has no dimensions metadata", () => {
db.prepare(`INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, 'clean')`).run(
"memory_vector_rebuild_v1",
);
expect(
resolvePersistedMemoryVectorIndexState({
db,
vectorTable: "memory_index_chunks_vec",
hasSemanticChunks: true,
}),
).toEqual({ state: "complete" });
});
it("keeps a pre-marker vector index unverified", () => {
expect(
resolvePersistedMemoryVectorIndexState({
db,
vectorTable: "memory_index_chunks_vec",
metaVectorDims: 768,
hasSemanticChunks: true,
}),
).toEqual({ state: "unverified" });
});
});

31
extensions/memory-core/src/memory/manager-vector-rebuild-state.ts
View File

@@ -1,6 +1,9 @@
// Memory Core plugin module owns persisted vector completeness state.
import type { DatabaseSync } from "node:sqlite";
import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import {
MEMORY_INDEX_META_TABLE,
type MemoryVectorIndexState,
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
const VECTOR_REBUILD_META_KEY = "memory_vector_rebuild_v1";
@@ -30,16 +33,36 @@ export function requiresMemoryVectorRebuild(params: {
metaVectorDims?: number;
hasSemanticChunks: boolean;
}): boolean {
const state = resolvePersistedMemoryVectorIndexState(params).state;
return state === "incomplete" || state === "unverified";
}
export function resolvePersistedMemoryVectorIndexState(params: {
db: DatabaseSync;
vectorTable: string;
metaVectorDims?: number;
hasSemanticChunks: boolean;
}): MemoryVectorIndexState {
const row = params.db
.prepare(`SELECT value FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ?`)
.get(VECTOR_REBUILD_META_KEY) as { value?: unknown } | undefined;
if (row?.value === "1") {
return true;
return { state: "incomplete" };
}
if (!vectorTableExists(params.db, params.vectorTable)) {
return Boolean(params.metaVectorDims && params.hasSemanticChunks);
return params.metaVectorDims && params.hasSemanticChunks
? { state: "incomplete" }
: { state: "empty" };
}
// The clean marker is published with the vector table. A later first
// incremental write can populate that table without rewriting vectorDims.
if (row?.value === "clean") {
return params.hasSemanticChunks ? { state: "complete" } : { state: "empty" };
}
if (params.hasSemanticChunks && !params.metaVectorDims) {
return { state: "incomplete" };
}
// Existing releases had no completeness marker. Rebuild their vector table
// once rather than assuming it has neither missing nor orphaned rows.
return row?.value !== "clean";
return { state: "unverified" };
}

7
extensions/memory-core/src/memory/manager.ts
View File

@@ -92,6 +92,7 @@ import {
runMemorySyncWithReadonlyRecovery,
type MemoryReadonlyRecoveryState,
} from "./manager-sync-control.js";
import { resolvePersistedMemoryVectorIndexState } from "./manager-vector-rebuild-state.js";
import { applyProjectRanking } from "./project-ranking.js";
import { applyTemporalDecayToHybridResults } from "./temporal-decay.js";
@@ -2194,6 +2195,12 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
: undefined,
vector: {
enabled: this.vector.enabled,
index: resolvePersistedMemoryVectorIndexState({
db: this.db,
vectorTable: VECTOR_TABLE,
metaVectorDims: this.vector.dims,
hasSemanticChunks: this.hasSemanticChunks(),
}),
storeAvailable: this.vector.available ?? undefined,
semanticAvailable: this.vector.semanticAvailable,
available: this.vector.semanticAvailable,

1
packages/memory-host-sdk/src/engine-storage.ts
View File

@@ -52,6 +52,7 @@ export type {
MemorySource,
MemorySyncParams,
MemorySyncProgressUpdate,
MemoryVectorIndexState,
} from "./host/types.js";
export {
dropMemoryPathFtsTriggers,

7
packages/memory-host-sdk/src/host/types.ts
View File

@@ -122,6 +122,12 @@ export type MemoryReadResult = {
};
/** Aggregated memory backend status for CLI/UI diagnostics. */
export type MemoryVectorIndexState =
| { state: "empty" }
| { state: "complete" }
| { state: "incomplete" }
| { state: "unverified" };
export type MemoryProviderStatus = {
backend: "builtin" | "qmd";
provider: string;
@@ -140,6 +146,7 @@ export type MemoryProviderStatus = {
fallback?: { from: string; reason?: string };
vector?: {
enabled: boolean;
index?: MemoryVectorIndexState;
storeAvailable?: boolean;
semanticAvailable?: boolean;
available?: boolean;

1
src/plugin-sdk/memory-core-host-engine-storage.ts
View File

@@ -87,4 +87,5 @@ export type {
ResolvedMemoryBackendConfig,
ResolvedQmdConfig,
ResolvedQmdMcporterConfig,
MemoryVectorIndexState,
} from "../../packages/memory-host-sdk/src/engine-storage.js";