diff --git a/extensions/qa-lab/src/scenario-runtime-api.test.ts b/extensions/qa-lab/src/scenario-runtime-api.test.ts index 5444bc57a501..0cb0bb72d98d 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.test.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.test.ts @@ -59,6 +59,7 @@ function createDeps(overrides?: Partial): QaScenarioRunti resolveGeneratedImagePath: fn, startAgentRun: fn, waitForAgentRun: fn, + waitForAgentHistoryReply: fn, listCronJobs: fn, waitForCronRunCompletion: fn, findManagedDreamingCronJob: fn, @@ -164,6 +165,7 @@ describe("createQaScenarioRuntimeApi", () => { expect(api.config).toEqual({ expected: "value" }); expect(api.waitForCondition).toBe(waitForCondition); expect(api.waitForChannelReady).toBe(api.waitForTransportReady); + expect(api.waitForAgentHistoryReply).toBe(deps.waitForAgentHistoryReply); expect(api.markGatewayLogCursor).toBe(deps.markGatewayLogCursor); expect(api.assertNoGatewayLogSentinels).toBe(deps.assertNoGatewayLogSentinels); expect(api.readSessionTranscriptSummary).toBe(deps.readSessionTranscriptSummary); diff --git a/extensions/qa-lab/src/scenario-runtime-api.ts b/extensions/qa-lab/src/scenario-runtime-api.ts index 3744c8c45a27..be0efbbf6972 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.ts @@ -71,6 +71,7 @@ export type QaScenarioRuntimeDeps = { resolveGeneratedImagePath: QaScenarioRuntimeFunction; startAgentRun: QaScenarioRuntimeFunction; waitForAgentRun: QaScenarioRuntimeFunction; + waitForAgentHistoryReply: QaScenarioRuntimeFunction; listCronJobs: QaScenarioRuntimeFunction; findManagedDreamingCronJob: QaScenarioRuntimeFunction; waitForCronRunCompletion: QaScenarioRuntimeFunction; @@ -164,6 +165,7 @@ type QaScenarioRuntimeApi< resolveGeneratedImagePath: TDeps["resolveGeneratedImagePath"]; startAgentRun: TDeps["startAgentRun"]; waitForAgentRun: TDeps["waitForAgentRun"]; + waitForAgentHistoryReply: TDeps["waitForAgentHistoryReply"]; listCronJobs: TDeps["listCronJobs"]; findManagedDreamingCronJob: TDeps["findManagedDreamingCronJob"]; waitForCronRunCompletion: TDeps["waitForCronRunCompletion"]; @@ -272,6 +274,7 @@ export function createQaScenarioRuntimeApi< resolveGeneratedImagePath: params.deps.resolveGeneratedImagePath, startAgentRun: params.deps.startAgentRun, waitForAgentRun: params.deps.waitForAgentRun, + waitForAgentHistoryReply: params.deps.waitForAgentHistoryReply, listCronJobs: params.deps.listCronJobs, findManagedDreamingCronJob: params.deps.findManagedDreamingCronJob, waitForCronRunCompletion: params.deps.waitForCronRunCompletion, diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index 1fb12624b1a4..2652f63b8f14 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -34,6 +34,7 @@ import { runQaCli, startAgentRun, waitForAgentRun, + waitForAgentHistoryReply, waitForMemorySearchMatch, } from "./suite-runtime-agent-process.js"; @@ -706,6 +707,38 @@ describe("qa suite runtime agent process helpers", () => { }); }); + it("waits for the latest assistant history reply", async () => { + const gatewayCall = vi + .fn() + .mockResolvedValueOnce({ messages: [{ role: "assistant", content: "still working" }] }) + .mockResolvedValueOnce({ + messages: [ + { role: "user", content: "hello" }, + { + role: "assistant", + content: [{ type: "output_text", text: "HISTORY-REPLY-OK" }], + }, + ], + }); + + await expect( + waitForAgentHistoryReply( + { gateway: { call: gatewayCall } } as never, + "session-history", + (text) => text === "HISTORY-REPLY-OK", + 1_000, + 1, + ), + ).resolves.toMatchObject({ + text: "HISTORY-REPLY-OK", + }); + expect(gatewayCall).toHaveBeenLastCalledWith( + "chat.history", + { sessionKey: "session-history", limit: 12 }, + { timeoutMs: 10_000 }, + ); + }); + it("waits for a specific agent run id", async () => { const gatewayCall = vi.fn(async () => ({ status: "ok" })); diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.ts b/extensions/qa-lab/src/suite-runtime-agent-process.ts index 660cbbbca8ac..0ff394dd1277 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.ts @@ -2,6 +2,7 @@ import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; import { randomUUID } from "node:crypto"; import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { @@ -14,6 +15,7 @@ import { readQaChildOutput, } from "./child-output.js"; import { QaSuiteInfraError } from "./errors.js"; +import { extractGatewayMessageText } from "./gateway-log-sentinel.js"; import { resolveQaNodeExecPath } from "./node-exec.js"; import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js"; import { waitForGatewayHealthy, waitForTransportReady } from "./suite-runtime-gateway.js"; @@ -35,11 +37,19 @@ type QaCronJob = { state?: { nextRunAtMs?: number }; }; +type QaChatHistoryResponse = { + messages?: unknown[]; +}; + const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\x1B\[[0-?]*[ -/]*[@-~]`, "g"); const MANAGED_DREAMING_CRON_MARKER = "[managed-by=memory-core.short-term-promotion]"; const MANAGED_DREAMING_CRON_NAME = "Memory Dreaming Promotion"; const MANAGED_DREAMING_PROMPT = "__openclaw_memory_core_short_term_promotion_dream__"; +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + function stripAnsiCodes(text: string) { return text.replace(ANSI_ESCAPE_PATTERN, ""); } @@ -368,6 +378,58 @@ async function waitForAgentRun( } } +function readLatestAssistantTextFromHistory(history: QaChatHistoryResponse | undefined) { + for (const message of [...(history?.messages ?? [])].reverse()) { + if (!isRecord(message) || message.role !== "assistant") { + continue; + } + const text = extractGatewayMessageText(message); + if (text) { + return text; + } + } + return undefined; +} + +async function readLatestAgentHistoryReply( + env: Pick, + sessionKey: string, +) { + const history = (await env.gateway.call( + "chat.history", + { + sessionKey, + limit: 12, + }, + { + timeoutMs: 10_000, + }, + )) as QaChatHistoryResponse | undefined; + return readLatestAssistantTextFromHistory(history); +} + +async function waitForAgentHistoryReply( + env: Pick, + sessionKey: string, + predicate: (text: string) => boolean | Promise, + timeoutMs = 30_000, + intervalMs = 250, +) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const text = await readLatestAgentHistoryReply(env, sessionKey); + if (text && (await predicate(text))) { + return { text }; + } + const remainingMs = timeoutMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + break; + } + await sleep(Math.min(intervalMs, remainingMs)); + } + throw new Error(`timed out after ${timeoutMs}ms`); +} + async function listCronJobs(env: Pick) { const payload = (await env.gateway.call( "cron.list", @@ -501,6 +563,7 @@ export { runAgentPrompt, runQaCli, startAgentRun, + waitForAgentHistoryReply, waitForMemorySearchMatch, waitForAgentRun, }; diff --git a/extensions/qa-lab/src/suite-runtime-agent.ts b/extensions/qa-lab/src/suite-runtime-agent.ts index 7fd7f86a18e1..0a11027ad1a9 100644 --- a/extensions/qa-lab/src/suite-runtime-agent.ts +++ b/extensions/qa-lab/src/suite-runtime-agent.ts @@ -14,6 +14,7 @@ export { runAgentPrompt, runQaCli, startAgentRun, + waitForAgentHistoryReply, waitForAgentRun, } from "./suite-runtime-agent-process.js"; export { diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index 914431b460d3..16d60b187e5a 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -29,6 +29,7 @@ const extractMediaPathFromText = vi.hoisted(() => vi.fn()); const resolveGeneratedImagePath = vi.hoisted(() => vi.fn()); const startAgentRun = vi.hoisted(() => vi.fn()); const waitForAgentRun = vi.hoisted(() => vi.fn()); +const waitForAgentHistoryReply = vi.hoisted(() => vi.fn()); const listCronJobs = vi.hoisted(() => vi.fn()); const findManagedDreamingCronJob = vi.hoisted(() => vi.fn()); const waitForCronRunCompletion = vi.hoisted(() => vi.fn()); @@ -98,6 +99,7 @@ vi.mock("./suite-runtime-agent.js", () => ({ resolveGeneratedImagePath, startAgentRun, waitForAgentRun, + waitForAgentHistoryReply, listCronJobs, findManagedDreamingCronJob, readDoctorMemoryStatus, @@ -255,6 +257,7 @@ describe("qa suite runtime flow", () => { findManagedDreamingCronJob: typeof findManagedDreamingCronJob; forceMemoryIndex: typeof forceMemoryIndex; runAgentPrompt: typeof runAgentPrompt; + waitForAgentHistoryReply: typeof waitForAgentHistoryReply; runRuntimeToolFixture: ( envArg: typeof env, configArg: Record, @@ -278,6 +281,7 @@ describe("qa suite runtime flow", () => { expect(call.deps.readSessionTranscriptSummary).toBe(readSessionTranscriptSummary); expect(call.deps.findManagedDreamingCronJob).toBe(findManagedDreamingCronJob); expect(call.deps.forceMemoryIndex).toBe(forceMemoryIndex); + expect(call.deps.waitForAgentHistoryReply).toBe(waitForAgentHistoryReply); expect(call.deps.runAgentPrompt).toBe(runAgentPrompt); await call.deps.runRuntimeToolFixture(env, { toolName: "read" }); expect(runRuntimeToolFixture).toHaveBeenCalledWith( diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 74fa9536d650..c6f775ebbadc 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -47,6 +47,7 @@ import { runAgentPrompt, runQaCli, startAgentRun, + waitForAgentHistoryReply, waitForAgentRun, writeWorkspaceSkill, } from "./suite-runtime-agent.js"; @@ -213,6 +214,7 @@ function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) { resolveGeneratedImagePath, startAgentRun, waitForAgentRun, + waitForAgentHistoryReply, listCronJobs, findManagedDreamingCronJob, waitForCronRunCompletion, diff --git a/qa/scenarios/agents/subagent-handoff.yaml b/qa/scenarios/agents/subagent-handoff.yaml index 207f802ac910..71f71219aec6 100644 --- a/qa/scenarios/agents/subagent-handoff.yaml +++ b/qa/scenarios/agents/subagent-handoff.yaml @@ -37,11 +37,14 @@ flow: expr: config.prompt timeoutMs: expr: liveTurnTimeoutMs(env, 90000) - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - agent:qa:subagent - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && (() => { const lower = normalizeLowercaseStringOrEmpty(candidate.text); return lower.includes('delegated task') && lower.includes('result') && lower.includes('evidence') && !lower.includes('waiting'); })()).at(-1)" + params: [text] + expr: "(() => { const lower = normalizeLowercaseStringOrEmpty(text); return lower.includes('delegated task') && lower.includes('result') && lower.includes('evidence') && !lower.includes('waiting'); })()" - expr: liveTurnTimeoutMs(env, 45000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - assert: diff --git a/qa/scenarios/memory/dreaming-shadow-trial-report.yaml b/qa/scenarios/memory/dreaming-shadow-trial-report.yaml index 5949f4b90c4c..fdd3c864a6dd 100644 --- a/qa/scenarios/memory/dreaming-shadow-trial-report.yaml +++ b/qa/scenarios/memory/dreaming-shadow-trial-report.yaml @@ -150,11 +150,14 @@ flow: expr: "String(memoryAfter) === config.seededMemory" message: expr: "`shadow trial modified durable memory instead of staying report-only: ${memoryAfter}`" - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - expr: config.sessionKey - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.safeMarker) && candidate.text.includes(config.reportName)).at(-1)" + params: [text] + expr: "text.includes(config.safeMarker) && text.includes(config.reportName)" - expr: liveTurnTimeoutMs(env, 30000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - assert: diff --git a/qa/scenarios/personal/failure-recovery.yaml b/qa/scenarios/personal/failure-recovery.yaml index 31bd6f8ee02e..5c1fcd2d3083 100644 --- a/qa/scenarios/personal/failure-recovery.yaml +++ b/qa/scenarios/personal/failure-recovery.yaml @@ -142,11 +142,14 @@ flow: - set: expectedReplyAll value: expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty) - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - expr: config.sessionKey - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(candidate.text).includes(needle))).at(-1)" + params: [text] + expr: "expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle))" - expr: liveTurnTimeoutMs(env, 30000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - set: normalizedReply diff --git a/qa/scenarios/personal/no-fake-progress.yaml b/qa/scenarios/personal/no-fake-progress.yaml index d7907a0ba799..992d67402dc7 100644 --- a/qa/scenarios/personal/no-fake-progress.yaml +++ b/qa/scenarios/personal/no-fake-progress.yaml @@ -138,11 +138,14 @@ flow: - set: expectedReplyAll value: expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty) - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - expr: config.sessionKey - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(candidate.text).includes(needle))).at(-1)" + params: [text] + expr: "expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle))" - expr: liveTurnTimeoutMs(env, 30000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - set: normalizedReply diff --git a/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml b/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml index 822bec010313..eae24389eb26 100644 --- a/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml +++ b/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml @@ -132,11 +132,14 @@ flow: expr: "!config.forbiddenNeedles.some((needle) => artifact.includes(needle))" message: expr: "`share-safe diagnostics artifact leaked unsafe source material: ${artifact}`" - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - expr: config.sessionKey - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.safeMarker) && candidate.text.includes(config.artifactName)).at(-1)" + params: [text] + expr: "text.includes(config.safeMarker) && text.includes(config.artifactName)" - expr: liveTurnTimeoutMs(env, 30000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - assert: diff --git a/qa/scenarios/workspace/long-running-release-audit.yaml b/qa/scenarios/workspace/long-running-release-audit.yaml index 94bbdc15af3b..d51edd90cd08 100644 --- a/qa/scenarios/workspace/long-running-release-audit.yaml +++ b/qa/scenarios/workspace/long-running-release-audit.yaml @@ -219,11 +219,14 @@ flow: expr: "JSON.stringify(report).includes('verified')" message: expr: "`report did not include a verification field: ${reportText}`" - - call: waitForCondition + - call: waitForAgentHistoryReply saveAs: outbound args: + - ref: env + - ref: sessionKey - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.trim() === 'RELEASE-AUDIT-COMPLETE').at(-1)" + params: [text] + expr: "text.trim() === 'RELEASE-AUDIT-COMPLETE'" - expr: liveTurnTimeoutMs(env, 45000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - call: readRawQaSessionStore