dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-08 11:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(qa): wait for live history replies in flow scenarios

Browse Source
This commit is contained in:
Vincent Koc
2026-06-23 02:47:47 +02:00
parent 5822e8074d
commit d716dfd532
13 changed files with 138 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/qa-lab/src/scenario-runtime-api.test.ts
View File

@@ -59,6 +59,7 @@ function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRunti
resolveGeneratedImagePath: fn,
startAgentRun: fn,
waitForAgentRun: fn,
waitForAgentHistoryReply: fn,
listCronJobs: fn,
waitForCronRunCompletion: fn,
findManagedDreamingCronJob: fn,
@@ -164,6 +165,7 @@ describe("createQaScenarioRuntimeApi", () => {
expect(api.config).toEqual({ expected: "value" });
expect(api.waitForCondition).toBe(waitForCondition);
expect(api.waitForChannelReady).toBe(api.waitForTransportReady);
expect(api.waitForAgentHistoryReply).toBe(deps.waitForAgentHistoryReply);
expect(api.markGatewayLogCursor).toBe(deps.markGatewayLogCursor);
expect(api.assertNoGatewayLogSentinels).toBe(deps.assertNoGatewayLogSentinels);
expect(api.readSessionTranscriptSummary).toBe(deps.readSessionTranscriptSummary);

3
extensions/qa-lab/src/scenario-runtime-api.ts
View File

@@ -71,6 +71,7 @@ export type QaScenarioRuntimeDeps = {
resolveGeneratedImagePath: QaScenarioRuntimeFunction;
startAgentRun: QaScenarioRuntimeFunction;
waitForAgentRun: QaScenarioRuntimeFunction;
waitForAgentHistoryReply: QaScenarioRuntimeFunction;
listCronJobs: QaScenarioRuntimeFunction;
findManagedDreamingCronJob: QaScenarioRuntimeFunction;
waitForCronRunCompletion: QaScenarioRuntimeFunction;
@@ -164,6 +165,7 @@ type QaScenarioRuntimeApi<
resolveGeneratedImagePath: TDeps["resolveGeneratedImagePath"];
startAgentRun: TDeps["startAgentRun"];
waitForAgentRun: TDeps["waitForAgentRun"];
waitForAgentHistoryReply: TDeps["waitForAgentHistoryReply"];
listCronJobs: TDeps["listCronJobs"];
findManagedDreamingCronJob: TDeps["findManagedDreamingCronJob"];
waitForCronRunCompletion: TDeps["waitForCronRunCompletion"];
@@ -272,6 +274,7 @@ export function createQaScenarioRuntimeApi<
resolveGeneratedImagePath: params.deps.resolveGeneratedImagePath,
startAgentRun: params.deps.startAgentRun,
waitForAgentRun: params.deps.waitForAgentRun,
waitForAgentHistoryReply: params.deps.waitForAgentHistoryReply,
listCronJobs: params.deps.listCronJobs,
findManagedDreamingCronJob: params.deps.findManagedDreamingCronJob,
waitForCronRunCompletion: params.deps.waitForCronRunCompletion,

33
extensions/qa-lab/src/suite-runtime-agent-process.test.ts
View File

@@ -34,6 +34,7 @@ import {
runQaCli,
startAgentRun,
waitForAgentRun,
waitForAgentHistoryReply,
waitForMemorySearchMatch,
} from "./suite-runtime-agent-process.js";
@@ -706,6 +707,38 @@ describe("qa suite runtime agent process helpers", () => {
});
});
it("waits for the latest assistant history reply", async () => {
const gatewayCall = vi
.fn()
.mockResolvedValueOnce({ messages: [{ role: "assistant", content: "still working" }] })
.mockResolvedValueOnce({
messages: [
{ role: "user", content: "hello" },
{
role: "assistant",
content: [{ type: "output_text", text: "HISTORY-REPLY-OK" }],
},
],
});
await expect(
waitForAgentHistoryReply(
{ gateway: { call: gatewayCall } } as never,
"session-history",
(text) => text === "HISTORY-REPLY-OK",
1_000,
1,
),
).resolves.toMatchObject({
text: "HISTORY-REPLY-OK",
});
expect(gatewayCall).toHaveBeenLastCalledWith(
"chat.history",
{ sessionKey: "session-history", limit: 12 },
{ timeoutMs: 10_000 },
);
});
it("waits for a specific agent run id", async () => {
const gatewayCall = vi.fn(async () => ({ status: "ok" }));

63
extensions/qa-lab/src/suite-runtime-agent-process.ts
View File

@@ -2,6 +2,7 @@
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
import {
@@ -14,6 +15,7 @@ import {
readQaChildOutput,
} from "./child-output.js";
import { QaSuiteInfraError } from "./errors.js";
import { extractGatewayMessageText } from "./gateway-log-sentinel.js";
import { resolveQaNodeExecPath } from "./node-exec.js";
import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js";
import { waitForGatewayHealthy, waitForTransportReady } from "./suite-runtime-gateway.js";
@@ -35,11 +37,19 @@ type QaCronJob = {
state?: { nextRunAtMs?: number };
};
type QaChatHistoryResponse = {
messages?: unknown[];
};
const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\x1B\[[0-?]*[ -/]*[@-~]`, "g");
const MANAGED_DREAMING_CRON_MARKER = "[managed-by=memory-core.short-term-promotion]";
const MANAGED_DREAMING_CRON_NAME = "Memory Dreaming Promotion";
const MANAGED_DREAMING_PROMPT = "__openclaw_memory_core_short_term_promotion_dream__";
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function stripAnsiCodes(text: string) {
return text.replace(ANSI_ESCAPE_PATTERN, "");
}
@@ -368,6 +378,58 @@ async function waitForAgentRun(
}
}
function readLatestAssistantTextFromHistory(history: QaChatHistoryResponse | undefined) {
for (const message of [...(history?.messages ?? [])].reverse()) {
if (!isRecord(message) || message.role !== "assistant") {
continue;
}
const text = extractGatewayMessageText(message);
if (text) {
return text;
}
}
return undefined;
}
async function readLatestAgentHistoryReply(
env: Pick<QaSuiteRuntimeEnv, "gateway">,
sessionKey: string,
) {
const history = (await env.gateway.call(
"chat.history",
{
sessionKey,
limit: 12,
},
{
timeoutMs: 10_000,
},
)) as QaChatHistoryResponse | undefined;
return readLatestAssistantTextFromHistory(history);
}
async function waitForAgentHistoryReply(
env: Pick<QaSuiteRuntimeEnv, "gateway">,
sessionKey: string,
predicate: (text: string) => boolean | Promise<boolean>,
timeoutMs = 30_000,
intervalMs = 250,
) {
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
const text = await readLatestAgentHistoryReply(env, sessionKey);
if (text && (await predicate(text))) {
return { text };
}
const remainingMs = timeoutMs - (Date.now() - startedAt);
if (remainingMs <= 0) {
break;
}
await sleep(Math.min(intervalMs, remainingMs));
}
throw new Error(`timed out after ${timeoutMs}ms`);
}
async function listCronJobs(env: Pick<QaSuiteRuntimeEnv, "gateway">) {
const payload = (await env.gateway.call(
"cron.list",
@@ -501,6 +563,7 @@ export {
runAgentPrompt,
runQaCli,
startAgentRun,
waitForAgentHistoryReply,
waitForMemorySearchMatch,
waitForAgentRun,
};

1
extensions/qa-lab/src/suite-runtime-agent.ts
View File

@@ -14,6 +14,7 @@ export {
runAgentPrompt,
runQaCli,
startAgentRun,
waitForAgentHistoryReply,
waitForAgentRun,
} from "./suite-runtime-agent-process.js";
export {

4
extensions/qa-lab/src/suite-runtime-flow.test.ts
View File

@@ -29,6 +29,7 @@ const extractMediaPathFromText = vi.hoisted(() => vi.fn());
const resolveGeneratedImagePath = vi.hoisted(() => vi.fn());
const startAgentRun = vi.hoisted(() => vi.fn());
const waitForAgentRun = vi.hoisted(() => vi.fn());
const waitForAgentHistoryReply = vi.hoisted(() => vi.fn());
const listCronJobs = vi.hoisted(() => vi.fn());
const findManagedDreamingCronJob = vi.hoisted(() => vi.fn());
const waitForCronRunCompletion = vi.hoisted(() => vi.fn());
@@ -98,6 +99,7 @@ vi.mock("./suite-runtime-agent.js", () => ({
resolveGeneratedImagePath,
startAgentRun,
waitForAgentRun,
waitForAgentHistoryReply,
listCronJobs,
findManagedDreamingCronJob,
readDoctorMemoryStatus,
@@ -255,6 +257,7 @@ describe("qa suite runtime flow", () => {
findManagedDreamingCronJob: typeof findManagedDreamingCronJob;
forceMemoryIndex: typeof forceMemoryIndex;
runAgentPrompt: typeof runAgentPrompt;
waitForAgentHistoryReply: typeof waitForAgentHistoryReply;
runRuntimeToolFixture: (
envArg: typeof env,
configArg: Record<string, unknown>,
@@ -278,6 +281,7 @@ describe("qa suite runtime flow", () => {
expect(call.deps.readSessionTranscriptSummary).toBe(readSessionTranscriptSummary);
expect(call.deps.findManagedDreamingCronJob).toBe(findManagedDreamingCronJob);
expect(call.deps.forceMemoryIndex).toBe(forceMemoryIndex);
expect(call.deps.waitForAgentHistoryReply).toBe(waitForAgentHistoryReply);
expect(call.deps.runAgentPrompt).toBe(runAgentPrompt);
await call.deps.runRuntimeToolFixture(env, { toolName: "read" });
expect(runRuntimeToolFixture).toHaveBeenCalledWith(

2
extensions/qa-lab/src/suite-runtime-flow.ts
View File

@@ -47,6 +47,7 @@ import {
runAgentPrompt,
runQaCli,
startAgentRun,
waitForAgentHistoryReply,
waitForAgentRun,
writeWorkspaceSkill,
} from "./suite-runtime-agent.js";
@@ -213,6 +214,7 @@ function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) {
resolveGeneratedImagePath,
startAgentRun,
waitForAgentRun,
waitForAgentHistoryReply,
listCronJobs,
findManagedDreamingCronJob,
waitForCronRunCompletion,

7
qa/scenarios/agents/subagent-handoff.yaml
View File

@@ -37,11 +37,14 @@ flow:
expr: config.prompt
timeoutMs:
expr: liveTurnTimeoutMs(env, 90000)
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- agent:qa:subagent
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && (() => { const lower = normalizeLowercaseStringOrEmpty(candidate.text); return lower.includes('delegated task') && lower.includes('result') && lower.includes('evidence') && !lower.includes('waiting'); })()).at(-1)"
params: [text]
expr: "(() => { const lower = normalizeLowercaseStringOrEmpty(text); return lower.includes('delegated task') && lower.includes('result') && lower.includes('evidence') && !lower.includes('waiting'); })()"
- expr: liveTurnTimeoutMs(env, 45000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- assert:

7
qa/scenarios/memory/dreaming-shadow-trial-report.yaml
View File

@@ -150,11 +150,14 @@ flow:
expr: "String(memoryAfter) === config.seededMemory"
message:
expr: "`shadow trial modified durable memory instead of staying report-only: ${memoryAfter}`"
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- expr: config.sessionKey
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.safeMarker) && candidate.text.includes(config.reportName)).at(-1)"
params: [text]
expr: "text.includes(config.safeMarker) && text.includes(config.reportName)"
- expr: liveTurnTimeoutMs(env, 30000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- assert:

7
qa/scenarios/personal/failure-recovery.yaml
View File

@@ -142,11 +142,14 @@ flow:
- set: expectedReplyAll
value:
expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty)
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- expr: config.sessionKey
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(candidate.text).includes(needle))).at(-1)"
params: [text]
expr: "expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle))"
- expr: liveTurnTimeoutMs(env, 30000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- set: normalizedReply

7
qa/scenarios/personal/no-fake-progress.yaml
View File

@@ -138,11 +138,14 @@ flow:
- set: expectedReplyAll
value:
expr: config.expectedReplyAll.map(normalizeLowercaseStringOrEmpty)
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- expr: config.sessionKey
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(candidate.text).includes(needle))).at(-1)"
params: [text]
expr: "expectedReplyAll.every((needle) => normalizeLowercaseStringOrEmpty(text).includes(needle))"
- expr: liveTurnTimeoutMs(env, 30000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- set: normalizedReply

7
qa/scenarios/personal/share-safe-diagnostics-artifact.yaml
View File

@@ -132,11 +132,14 @@ flow:
expr: "!config.forbiddenNeedles.some((needle) => artifact.includes(needle))"
message:
expr: "`share-safe diagnostics artifact leaked unsafe source material: ${artifact}`"
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- expr: config.sessionKey
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.safeMarker) && candidate.text.includes(config.artifactName)).at(-1)"
params: [text]
expr: "text.includes(config.safeMarker) && text.includes(config.artifactName)"
- expr: liveTurnTimeoutMs(env, 30000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- assert:

7
qa/scenarios/workspace/long-running-release-audit.yaml
View File

@@ -219,11 +219,14 @@ flow:
expr: "JSON.stringify(report).includes('verified')"
message:
expr: "`report did not include a verification field: ${reportText}`"
- call: waitForCondition
- call: waitForAgentHistoryReply
saveAs: outbound
args:
- ref: env
- ref: sessionKey
- lambda:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.trim() === 'RELEASE-AUDIT-COMPLETE').at(-1)"
params: [text]
expr: "text.trim() === 'RELEASE-AUDIT-COMPLETE'"
- expr: liveTurnTimeoutMs(env, 45000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- call: readRawQaSessionStore