dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-13 17:07:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): add stable mock request cursors

Browse Source
This commit is contained in:
Vincent Koc
2026-07-12 07:58:35 +02:00
parent 0d84d691e5
commit bdc9c6070d
32 changed files with 462 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
extensions/qa-lab/src/providers/aimock/server.test.ts
View File

@@ -102,6 +102,72 @@ describe("qa aimock server", () => {
}
});
it("reads requests after a stable debug cursor", async () => {
const server = await startQaAimockServer({
host: "127.0.0.1",
port: 0,
});
const post = async (text: string) => {
const response = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
model: "aimock/gpt-5.6-luna",
stream: false,
input: [makeResponsesInput(text)],
}),
});
expect(response.status).toBe(200);
};
try {
expect(
await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => response.json()),
).toEqual({ cursor: 0 });
const debugRequestLimit = 1_000;
for (let index = 0; index < debugRequestLimit; index += 1) {
await post(`aimock cursor ${index}`);
}
const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) =>
response.json(),
);
expect(cursor).toEqual({ cursor: debugRequestLimit });
await post("aimock cursor overflow");
const retained = (await fetch(`${server.baseUrl}/debug/requests`).then((response) =>
response.json(),
)) as Array<{ prompt?: unknown }>;
expect(retained).toHaveLength(debugRequestLimit);
expect(retained[0]?.prompt).toBe("aimock cursor 1");
expect(retained.at(-1)?.prompt).toBe("aimock cursor overflow");
const after = await fetch(`${server.baseUrl}/debug/requests?after=${debugRequestLimit}`);
expect(after.status).toBe(200);
const requests = (await after.json()) as Array<{ prompt?: unknown }>;
expect(requests).toHaveLength(1);
expect(requests[0]?.prompt).toBe("aimock cursor overflow");
const expired = await fetch(`${server.baseUrl}/debug/requests?after=0`);
expect(expired.status).toBe(409);
expect(await expired.json()).toEqual({
error: "request cursor expired",
after: 0,
oldestCursor: 2,
latestCursor: debugRequestLimit + 1,
});
const futureCursor = debugRequestLimit + 2;
const future = await fetch(`${server.baseUrl}/debug/requests?after=${futureCursor}`);
expect(future.status).toBe(409);
expect(await future.json()).toEqual({
error: "request cursor is ahead of the latest recorded request",
after: futureCursor,
latestCursor: debugRequestLimit + 1,
});
} finally {
await server.stop();
}
});
it("treats OpenAI Codex model refs as OpenAI-compatible snapshots", async () => {
const server = await startQaAimockServer({
host: "127.0.0.1",

93
extensions/qa-lab/src/providers/aimock/server.ts
View File

@@ -1,11 +1,13 @@
// Qa Lab plugin module implements server behavior.
import type { IncomingMessage, ServerResponse } from "node:http";
import {
type Journal,
LLMock,
type ChatCompletionRequest,
type JournalEntry,
type Mountable,
} from "@copilotkit/aimock";
import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js";
import { writeJson } from "../shared/http-json.js";
type AimockRequestSnapshot = {
@@ -23,6 +25,8 @@ type AimockRequestSnapshot = {
toolOutputStructuredError?: true;
};
const AIMOCK_DEBUG_REQUEST_LIMIT = 1_000;
// Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts.
// This mock mirrors the wire shape so delimiter drift fails through QA timeouts.
const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>";
@@ -217,18 +221,96 @@ function toRequestSnapshots(entries: JournalEntry[]): AimockRequestSnapshot[] {
return snapshots;
}
function createDebugMount(mock: LLMock): Mountable {
function createDebugMount(): Mountable {
let journal: Journal | undefined;
let nextRequestCursor = 1;
const requestCursors = new Map<string, number>();
return {
async handleRequest(_req: IncomingMessage, res: ServerResponse, pathname: string) {
const entries = mock.getRequests();
setJournal(nextJournal) {
if (journal === nextJournal) {
return;
}
if (journal) {
throw new Error("AIMock debug request cursor journal changed unexpectedly");
}
journal = nextJournal;
const addJournalEntry = journal.add.bind(journal);
// AIMock evicts its request journal FIFO. Assign cursors at insertion time
// so the debug boundary remains monotonic after retained entries rotate.
journal.add = (entry) => {
const recorded = addJournalEntry(entry);
requestCursors.set(recorded.id, nextRequestCursor++);
if (requestCursors.size > AIMOCK_DEBUG_REQUEST_LIMIT) {
const oldestRequestId = requestCursors.keys().next().value;
if (oldestRequestId !== undefined) {
requestCursors.delete(oldestRequestId);
}
}
return recorded;
};
},
async handleRequest(req: IncomingMessage, res: ServerResponse, pathname: string) {
const entries = journal?.getAll() ?? [];
const snapshots = toRequestSnapshots(entries);
const cursorSnapshots = entries.map((entry, index) => {
const cursor = requestCursors.get(entry.id);
if (cursor === undefined) {
throw new Error(`AIMock debug request cursor missing for ${entry.id}`);
}
const snapshot = snapshots[index];
if (!snapshot) {
throw new Error(`AIMock debug request snapshot missing for ${entry.id}`);
}
return { cursor, snapshot };
});
const url = new URL(req.url ?? "/", "http://127.0.0.1");
if (pathname === "/last-request") {
const lastSnapshot = snapshots.at(-1);
writeJson(res, 200, lastSnapshot ?? { ok: false, error: "no request recorded" });
return true;
}
if (pathname === "/request-cursor") {
writeJson(res, 200, { cursor: nextRequestCursor - 1 });
return true;
}
if (pathname === "/requests") {
writeJson(res, 200, snapshots);
const afterText = url.searchParams.get("after");
if (afterText === null) {
writeJson(res, 200, snapshots);
return true;
}
const after = parseQaDebugRequestCursor(afterText);
if (after === null) {
writeJson(res, 400, { error: "after must be a non-negative safe integer" });
return true;
}
const latestCursor = nextRequestCursor - 1;
const oldestCursor = cursorSnapshots[0]?.cursor ?? nextRequestCursor;
if (after > latestCursor) {
writeJson(res, 409, {
error: "request cursor is ahead of the latest recorded request",
after,
latestCursor,
});
return true;
}
if (after < oldestCursor - 1) {
writeJson(res, 409, {
error: "request cursor expired",
after,
oldestCursor,
latestCursor,
});
return true;
}
writeJson(
res,
200,
cursorSnapshots
.filter((request) => request.cursor > after)
.map((request) => request.snapshot),
);
return true;
}
if (pathname === "/image-generations") {
@@ -252,9 +334,10 @@ export async function startQaAimockServer(params?: { host?: string; port?: numbe
port: params?.port ?? 0,
strict: false,
logLevel: "silent",
journalMaxEntries: AIMOCK_DEBUG_REQUEST_LIMIT,
});
mock.mount("/debug", createDebugMount(mock));
mock.mount("/debug", createDebugMount());
mock.onMessage(/.*/, { content: "AIMOCK_QA_OK" });
await mock.start();

70
extensions/qa-lab/src/providers/mock-openai/server.test.ts
View File

@@ -232,6 +232,76 @@ function explicitSessionsSpawnPrompt(token: string) {
}
describe("qa mock openai server", () => {
it("keeps cursor reads correct when retained debug requests rotate", async () => {
const server = await startMockServer();
const debugRequestLimit = 2_000;
const readCursor = async () =>
requireRecord(
await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => response.json()),
"debug request cursor",
).cursor;
expect(await readCursor()).toBe(0);
for (let index = 0; index < debugRequestLimit; index += 1) {
await expectResponsesJson(server, {
stream: false,
model: "gpt-5.6-luna",
input: [makeUserInput(`cursor request ${index}`)],
});
}
const cursor = await readCursor();
expect(cursor).toBe(debugRequestLimit);
await expectResponsesJson(server, {
stream: false,
model: "gpt-5.6-luna",
input: [makeUserInput("cursor request overflow")],
});
const retained = requireArray(
await fetch(`${server.baseUrl}/debug/requests`).then((response) => response.json()),
"retained debug requests",
);
expect(retained).toHaveLength(debugRequestLimit);
expect(requireRecord(retained[0], "retained request 0").cursor).toBe(2);
expect(requireRecord(retained.at(-1), "last retained request").cursor).toBe(
debugRequestLimit + 1,
);
const nextRequests = requireArray(
await fetch(`${server.baseUrl}/debug/requests?after=${cursor}`).then((response) =>
response.json(),
),
"debug requests after cursor",
);
expect(nextRequests).toHaveLength(1);
expect(String(requireRecord(nextRequests[0], "next request").prompt)).toContain("overflow");
const expired = await fetch(`${server.baseUrl}/debug/requests?after=0`);
expect(expired.status).toBe(409);
expect(await expired.json()).toEqual({
error: "request cursor expired",
after: 0,
oldestCursor: 2,
latestCursor: debugRequestLimit + 1,
});
const futureCursor = debugRequestLimit + 2;
const future = await fetch(`${server.baseUrl}/debug/requests?after=${futureCursor}`);
expect(future.status).toBe(409);
expect(await future.json()).toEqual({
error: "request cursor is ahead of the latest recorded request",
after: futureCursor,
latestCursor: debugRequestLimit + 1,
});
const invalid = await fetch(`${server.baseUrl}/debug/requests?after=1.5`);
expect(invalid.status).toBe(400);
expect(await invalid.json()).toEqual({
error: "after must be a non-negative safe integer",
});
});
it("retains enough debug requests for long shared QA runs", async () => {
const server = await startMockServer();

69
extensions/qa-lab/src/providers/mock-openai/server.ts
View File

@@ -6,6 +6,7 @@ import { escapeRegExp, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utilit
import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress";
import { closeQaHttpServer } from "../../bus-server.js";
import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js";
import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js";
import { writeJson } from "../shared/http-json.js";
type ResponsesInputItem = Record<string, unknown>;
@@ -97,6 +98,7 @@ export function resolveProviderVariant(model: string | undefined): MockOpenAiPro
}
type MockOpenAiRequestSnapshot = {
cursor: number;
raw: string;
body: Record<string, unknown>;
prompt: string;
@@ -113,6 +115,8 @@ type MockOpenAiRequestSnapshot = {
toolOutputStructuredError?: true;
};
type MockOpenAiRequestSnapshotInput = Omit<MockOpenAiRequestSnapshot, "cursor">;
// Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts.
// This mock mirrors the wire shape so delimiter drift fails through QA timeouts.
const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>";
@@ -3784,6 +3788,16 @@ export async function startQaMockOpenAiServer(params?: {
};
let lastRequest: MockOpenAiRequestSnapshot | null = null;
const requests: MockOpenAiRequestSnapshot[] = [];
let nextRequestCursor = 1;
const recordRequest = (snapshot: MockOpenAiRequestSnapshotInput) => {
const recorded = { ...snapshot, cursor: nextRequestCursor++ };
lastRequest = recorded;
requests.push(recorded);
if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) {
requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT);
}
return recorded;
};
const inflightRequests = new Map<number, { prompt: string; allInputText: string }>();
let nextInflightRequestId = 1;
const imageGenerationRequests: Array<Record<string, unknown>> = [];
@@ -3812,8 +3826,45 @@ export async function startQaMockOpenAiServer(params?: {
writeJson(res, 200, lastRequest ?? { ok: false, error: "no request recorded" });
return;
}
if (req.method === "GET" && url.pathname === "/debug/request-cursor") {
writeJson(res, 200, { cursor: nextRequestCursor - 1 });
return;
}
if (req.method === "GET" && url.pathname === "/debug/requests") {
writeJson(res, 200, requests);
const afterText = url.searchParams.get("after");
if (afterText === null) {
writeJson(res, 200, requests);
return;
}
const after = parseQaDebugRequestCursor(afterText);
if (after === null) {
writeJson(res, 400, { error: "after must be a non-negative safe integer" });
return;
}
const latestCursor = nextRequestCursor - 1;
const oldestCursor = requests[0]?.cursor ?? nextRequestCursor;
if (after > latestCursor) {
writeJson(res, 409, {
error: "request cursor is ahead of the latest recorded request",
after,
latestCursor,
});
return;
}
if (after < oldestCursor - 1) {
writeJson(res, 409, {
error: "request cursor expired",
after,
oldestCursor,
latestCursor,
});
return;
}
writeJson(
res,
200,
requests.filter((request) => request.cursor > after),
);
return;
}
if (req.method === "GET" && url.pathname === "/debug/inflight-requests") {
@@ -3897,7 +3948,7 @@ export async function startQaMockOpenAiServer(params?: {
inflightRequests.delete(inflightRequestId);
}
const resolvedModel = typeof body.model === "string" ? body.model : "";
lastRequest = {
recordRequest({
raw,
body,
prompt,
@@ -3912,11 +3963,7 @@ export async function startQaMockOpenAiServer(params?: {
plannedToolArgs: extractPlannedToolArgs(events),
toolOutputCallId: extractToolOutputCallId(input) || undefined,
...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}),
};
requests.push(lastRequest);
if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) {
requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT);
}
});
if (body.stream === false) {
const completion = events.at(-1);
if (!completion || completion.type !== "response.completed") {
@@ -3959,7 +4006,7 @@ export async function startQaMockOpenAiServer(params?: {
// is what lets a single parity run diff assertions across both lanes.
// Reuse the normalized model so an empty-string body.model no longer
// leaks through to `lastRequest.model`.
lastRequest = {
recordRequest({
raw,
body: body as Record<string, unknown>,
prompt: extractLastUserText(input),
@@ -3973,11 +4020,7 @@ export async function startQaMockOpenAiServer(params?: {
plannedToolArgs: extractPlannedToolArgs(events),
toolOutputCallId: extractToolOutputCallId(input) || undefined,
...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}),
};
requests.push(lastRequest);
if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) {
requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT);
}
});
if (body.stream === true) {
writeAnthropicSse(res, streamEvents);
return;

27
extensions/qa-lab/src/providers/shared/debug-request-cursor.ts Normal file
View File

@@ -0,0 +1,27 @@
// Shared by the QA mock providers and the fixtures that consume their debug logs.
export function parseQaDebugRequestCursor(value: string): number | null {
if (!/^(?:0|[1-9]\d*)$/u.test(value)) {
return null;
}
const cursor = Number(value);
return Number.isSafeInteger(cursor) ? cursor : null;
}
export function readQaMockRequestCursor(value: unknown): number {
const cursor =
value && typeof value === "object" && !Array.isArray(value) && "cursor" in value
? value.cursor
: undefined;
if (typeof cursor !== "number" || !Number.isSafeInteger(cursor) || cursor < 0) {
throw new Error("mock provider request cursor response was invalid");
}
return cursor;
}
export function qaMockRequestCursorUrl(baseUrl: string): string {
return `${baseUrl.replace(/\/+$/u, "")}/debug/request-cursor`;
}
export function qaMockRequestsAfterUrl(baseUrl: string, cursor: number): string {
return `${baseUrl.replace(/\/+$/u, "")}/debug/requests?after=${cursor}`;
}

32
extensions/qa-lab/src/runtime-tool-fixture.test.ts
View File

@@ -118,7 +118,7 @@ async function runMockRuntimeToolFixtureWithOutputs(params: {
const failureCallId = `call-${params.toolName}-failure`;
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: promptSnippet,
@@ -526,7 +526,7 @@ describe("runtime tool fixture", () => {
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -577,7 +577,7 @@ describe("runtime tool fixture", () => {
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -631,7 +631,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -678,7 +678,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -733,7 +733,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -789,7 +789,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -838,7 +838,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -890,7 +890,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -942,7 +942,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -989,7 +989,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=image_generate",
@@ -1036,7 +1036,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -1091,7 +1091,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -1199,7 +1199,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -1254,7 +1254,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",
@@ -1309,7 +1309,7 @@ describe("runtime tool fixture", () => {
});
const fetchJson = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValueOnce({ cursor: 0 })
.mockResolvedValueOnce([
{
allInputText: "target=read",

36
extensions/qa-lab/src/runtime-tool-fixture.ts
View File

@@ -3,6 +3,11 @@ import fs from "node:fs/promises";
import path from "node:path";
import { loadTranscriptEventsSync } from "openclaw/plugin-sdk/session-store-runtime";
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
import {
qaMockRequestCursorUrl,
qaMockRequestsAfterUrl,
readQaMockRequestCursor,
} from "./providers/shared/debug-request-cursor.js";
import {
type QaRuntimeToolCoverageMetadata,
readRuntimeToolCoverageMetadata,
@@ -464,31 +469,27 @@ function requestLinksPlannedToolOutput(
function findPlannedRequest(params: {
requests: readonly QaRuntimeToolFixtureRequest[];
requestCountBefore: number;
promptSnippet: string;
excludedPromptSnippet?: string;
toolName: string;
}) {
return params.requests
.slice(params.requestCountBefore)
.find(
(request) =>
requestMatchesPrompt(request, params.promptSnippet) &&
(!params.excludedPromptSnippet ||
!requestMatchesPrompt(request, params.excludedPromptSnippet)) &&
request.plannedToolName === params.toolName,
);
return params.requests.find(
(request) =>
requestMatchesPrompt(request, params.promptSnippet) &&
(!params.excludedPromptSnippet ||
!requestMatchesPrompt(request, params.excludedPromptSnippet)) &&
request.plannedToolName === params.toolName,
);
}
function findExecutedRequest(params: {
requests: readonly QaRuntimeToolFixtureRequest[];
requestCountBefore: number;
promptSnippet: string;
excludedPromptSnippet?: string;
toolName: string;
}) {
let plannedRequest: QaRuntimeToolFixtureRequest | undefined;
for (const request of params.requests.slice(params.requestCountBefore)) {
for (const request of params.requests) {
if (!requestMatchesPrompt(request, params.promptSnippet)) {
continue;
}
@@ -667,9 +668,8 @@ export async function runRuntimeToolFixture(
`failure target=${toolName}`,
);
const happyPathOutputRequired = readBoolean(config.happyPathOutputRequired, true);
const requestCountBefore = env.mock
? readQaRuntimeToolFixtureRequests(await deps.fetchJson(`${env.mock.baseUrl}/debug/requests`))
.length
const requestCursorBefore = env.mock
? readQaMockRequestCursor(await deps.fetchJson(qaMockRequestCursorUrl(env.mock.baseUrl)))
: 0;
await deps.runAgentPrompt(env, {
@@ -744,31 +744,27 @@ export async function runRuntimeToolFixture(
}
const requests = readQaRuntimeToolFixtureRequests(
await deps.fetchJson(`${env.mock.baseUrl}/debug/requests`),
await deps.fetchJson(qaMockRequestsAfterUrl(env.mock.baseUrl, requestCursorBefore)),
);
const happyPlannedRequest = findPlannedRequest({
requests,
requestCountBefore,
promptSnippet,
excludedPromptSnippet: failurePromptSnippet,
toolName,
});
const happyRequest = findExecutedRequest({
requests,
requestCountBefore,
promptSnippet,
excludedPromptSnippet: failurePromptSnippet,
toolName,
});
const failurePlannedRequest = findPlannedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});
const failureRequest = findExecutedRequest({
requests,
requestCountBefore,
promptSnippet: failurePromptSnippet,
toolName,
});

23
extensions/qa-lab/src/tool-search-gateway.fixture.test.ts
View File

@@ -10,6 +10,11 @@ import {
outputText,
outputToolNames,
} from "./fixture-utils.js";
import {
qaMockRequestCursorUrl,
qaMockRequestsAfterUrl,
readQaMockRequestCursor,
} from "./providers/shared/debug-request-cursor.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import {
assertToolSearchLaneResults,
@@ -23,6 +28,22 @@ afterEach(() => {
});
describe("tool search gateway e2e fetch helper", () => {
it("builds and validates mock request cursor reads", () => {
expect(readQaMockRequestCursor({ cursor: 42 })).toBe(42);
expect(qaMockRequestCursorUrl("http://mock.test/")).toBe(
"http://mock.test/debug/request-cursor",
);
expect(qaMockRequestsAfterUrl("http://mock.test/", 42)).toBe(
"http://mock.test/debug/requests?after=42",
);
expect(() => readQaMockRequestCursor({ cursor: -1 })).toThrow(
"mock provider request cursor response was invalid",
);
expect(() => readQaMockRequestCursor([])).toThrow(
"mock provider request cursor response was invalid",
);
});
it("rejects loose numeric env limits instead of parsing prefixes", () => {
expect(() =>
readToolSearchGatewayFetchLimits({
@@ -222,7 +243,7 @@ describe("tool search gateway e2e lane result", () => {
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce(jsonResponse([]))
.mockResolvedValueOnce(jsonResponse({ cursor: 0 }))
.mockResolvedValueOnce(jsonResponse({ output: [], status: "completed" }))
.mockResolvedValueOnce(
jsonResponse([

14
extensions/qa-lab/src/tool-search-gateway.fixture.ts
View File

@@ -14,6 +14,11 @@ import {
subtractMentionCounts,
type QaFixtureFetchJsonOptions,
} from "./fixture-utils.js";
import {
qaMockRequestCursorUrl,
qaMockRequestsAfterUrl,
readQaMockRequestCursor,
} from "./providers/shared/debug-request-cursor.js";
import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
@@ -371,7 +376,9 @@ export async function runToolSearchGatewayLane(params: {
stateDir,
targetTool: params.fixture.targetTool,
});
const beforeRequests = (await fetchJson(`${providerBaseUrl}/debug/requests`)) as unknown[];
const requestCursorBefore = readQaMockRequestCursor(
await fetchJson(qaMockRequestCursorUrl(providerBaseUrl)),
);
const response = await fetchJson(
`${params.env.gateway.baseUrl}/v1/responses`,
{
@@ -403,7 +410,9 @@ export async function runToolSearchGatewayLane(params: {
},
{ timeoutMs: liveTurnTimeoutMs(params.env, 30_000) },
);
const requests = (await fetchJson(`${providerBaseUrl}/debug/requests`)) as Array<{
const laneRequests = (await fetchJson(
qaMockRequestsAfterUrl(providerBaseUrl, requestCursorBefore),
)) as Array<{
raw?: string;
body?: { tools?: unknown[] };
instructions?: string;
@@ -412,7 +421,6 @@ export async function runToolSearchGatewayLane(params: {
toolOutput?: string;
plannedToolName?: string;
}>;
const laneRequests = requests.slice(beforeRequests.length);
const lastRequest = laneRequests.at(-1) ?? {};
const responseStatus = (response as { status?: unknown }).status;
const mentionCountsAfter = await countToolSearchSessionLogMentions({

10
qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml
View File

@@ -70,9 +70,9 @@ flow:
- set: targetSessionKey
value:
expr: "buildAgentSessionKey({ agentId: 'qa', channel: 'qa-channel', accountId: 'default', peer: { kind: 'group', id: `group:${config.conversationId}` }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })"
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: state.addInboundMessage
args:
- conversation:
@@ -91,7 +91,7 @@ flow:
- lambda:
async: true
params: []
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0) : true"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0) : true"
- expr: liveTurnTimeoutMs(env, 60000)
- 500
- call: waitForCondition
@@ -100,7 +100,7 @@ flow:
- lambda:
async: true
params: []
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.targetPromptSnippet) && request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker) : true"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.targetPromptSnippet) && request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker) : true"
- expr: liveTurnTimeoutMs(env, 90000)
- 500
- call: waitForOutboundMessage
@@ -128,7 +128,7 @@ flow:
expr: "`expected exactly one requester-visible A2A marker after duplicate window, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: config.conversationId })}`"
- set: scenarioRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ prompt: String(request.prompt ?? '').slice(0, 220), plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 220) : null })) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).map((request) => ({ prompt: String(request.prompt ?? '').slice(0, 220), plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 220) : null })) : []"
- assert:
expr: "!env.mock || scenarioRequests.filter((request) => request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0).length === 1"
message:

6
qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml
View File

@@ -54,9 +54,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- sendInbound:
conversation:
id:
@@ -84,7 +84,7 @@ flow:
expr: "`expected exactly one fallback group reply, saw ${matchingOutbound.length}`"
- set: scenarioRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
- assert:
expr: "!env.mock || scenarioRequests.length > 0"
message: expected mock request evidence for fallback scenario

8
qa/scenarios/channels/group-visible-reply-tool.yaml
View File

@@ -47,9 +47,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- sendInbound:
conversation:
id:
@@ -65,11 +65,11 @@ flow:
- lambda:
async: true
params: []
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true"
- expr: liveTurnTimeoutMs(env, 180000)
- set: scenarioRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
- assert:
expr: "!env.mock || scenarioRequests.some((request) => request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker)"
message:

6
qa/scenarios/channels/message-tool-stranded-final-reply.yaml
View File

@@ -55,9 +55,9 @@ flow:
- set: logCursor
value:
expr: markGatewayLogCursor()
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- sendInbound:
conversation:
id:
@@ -78,7 +78,7 @@ flow:
- expr: liveTurnTimeoutMs(env, 180000)
- set: scenarioRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []"
- set: strandedRequests
value:
expr: "scenarioRequests.filter((request) => !String(request.allInputText ?? '').includes(config.retryPromptNeedle))"

8
qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml
View File

@@ -52,9 +52,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- sendInbound:
conversation:
id:
@@ -75,7 +75,7 @@ flow:
- expr: liveTurnTimeoutMs(env, 180000)
- set: scenarioRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"
- set: retryRequests
value:
expr: "scenarioRequests.filter((request) => String(request.allInputText ?? '').includes(config.retryPromptNeedle))"
@@ -110,7 +110,7 @@ flow:
- expr: liveTurnTimeoutMs(env, 8000)
- set: settledRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"
- set: settledRetryRequests
value:
expr: "settledRequests.filter((request) => String(request.allInputText ?? '').includes(config.retryPromptNeedle))"

8
qa/scenarios/channels/webchat-direct-reply-routing.yaml
View File

@@ -56,9 +56,9 @@ flow:
- set: startIndex
value:
expr: state.getSnapshot().messages.length
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- sendInbound:
conversation:
id:
@@ -76,7 +76,7 @@ flow:
args:
- lambda:
async: true
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true"
- expr: liveTurnTimeoutMs(env, 60000)
- 500
- call: waitForOutboundMessage
@@ -93,7 +93,7 @@ flow:
catch:
- set: directReplyDebugRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []"
- throw:
expr: "`direct reply marker missing: ${directReplyError?.message ?? directReplyError}; transcript=${formatTransportTranscript(state, { conversationId })}; requests=${JSON.stringify(directReplyDebugRequests)}`"
- call: waitForCondition

6
qa/scenarios/goals/goal-context-next-turn.yaml
View File

@@ -76,9 +76,9 @@ flow:
timeoutMs:
expr: liveTurnTimeoutMs(env, 60000)
saveAs: goalStartReply
- set: requestCountBeforeSecondTurn
- set: requestCursorBeforeSecondTurn
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
- set: secondOutboundIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
@@ -97,7 +97,7 @@ flow:
args:
- lambda:
async: true
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeSecondTurn).find((request) => String(request.prompt ?? '').includes(config.secondMessage))"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeSecondTurn}`)).find((request) => String(request.prompt ?? '').includes(config.secondMessage))"
- expr: liveTurnTimeoutMs(env, 60000)
- 100
- waitForOutbound:

6
qa/scenarios/goals/goal-context-survives-compaction.yaml
View File

@@ -133,9 +133,9 @@ flow:
expr: "!normalizeLowercaseStringOrEmpty(compactReply.text).includes('skipped')"
message:
expr: "`manual compaction did not compact the seeded session: ${compactReply.text}`"
- set: requestCountBeforePostCompactionTurn
- set: requestCursorBeforePostCompactionTurn
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
- set: postCompactOutboundIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
@@ -154,7 +154,7 @@ flow:
args:
- lambda:
async: true
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforePostCompactionTurn).find((request) => String(request.prompt ?? '').includes(config.postCompactionMessage))"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforePostCompactionTurn}`)).find((request) => String(request.prompt ?? '').includes(config.postCompactionMessage))"
- expr: liveTurnTimeoutMs(env, 60000)
- 100
- waitForOutbound:

12
qa/scenarios/memory/active-memory-preprompt-recall.yaml
View File

@@ -122,9 +122,9 @@ flow:
expr: "String(doctorFixOutput).includes('Migrated 1 Active Memory session toggle entry')"
message:
expr: "`doctor --fix did not migrate the Active Memory session toggle: ${doctorFixOutput}`"
- set: requestCountBeforeBaseline
- set: requestCursorBeforeBaseline
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: baselineStartIndex
value:
expr: "state.getSnapshot().messages.length"
@@ -155,16 +155,16 @@ flow:
then:
- set: baselineMockRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeBaseline)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeBaseline}`))"
- set: baselineSessionStore
value:
expr: "await readRawQaSessionStore(env)"
- assert:
expr: "!Array.isArray(baselineSessionStore[baselineSessionKey]?.pluginDebugEntries) || !baselineSessionStore[baselineSessionKey].pluginDebugEntries.some((pluginEntry) => pluginEntry?.pluginId === 'active-memory')"
message: baseline session unexpectedly recorded active-memory plugin activity
- set: requestCountBeforeActive
- set: requestCursorBeforeActive
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: activeStartIndex
value:
expr: "state.getSnapshot().messages.length"
@@ -227,7 +227,7 @@ flow:
then:
- set: mockRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeActive)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeActive}`))"
- assert:
expr: "mockRequests.some((request) => request.allInputText.includes('You are a memory search agent.') && request.plannedToolName === 'memory_search')"
message: expected mock Active Memory search request

6
qa/scenarios/memory/dreaming-shadow-trial-report.yaml
View File

@@ -120,9 +120,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -172,7 +172,7 @@ flow:
expr: "`shadow trial reply overclaimed promotion: ${outbound.text}`"
- set: shadowTrialDebugRequests
value:
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /dreaming shadow trial report check/i.test(String(request.allInputText ?? ''))) : []"
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /dreaming shadow trial report check/i.test(String(request.allInputText ?? ''))) : []"
- assert:
expr: "!env.mock || shadowTrialDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
message:

12
qa/scenarios/personal/approval-denial-stop.yaml
View File

@@ -58,9 +58,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -79,7 +79,7 @@ flow:
expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.setupMarker)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.setupPrompt)).some((request) => request.plannedToolName)"
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.setupPrompt)).some((request) => request.plannedToolName)"
message: setup turn should not plan a tool before any denied request
- set: beforeDenialCursor
value:
@@ -87,9 +87,9 @@ flow:
- set: beforeDenialOutboundCursor
value:
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound').length"
- set: denialRequestStart
- set: denialRequestCursor
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -110,7 +110,7 @@ flow:
- sinceIndex:
ref: beforeDenialOutboundCursor
- assert:
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(denialRequestStart).filter((request) => String(request.allInputText ?? '').includes(config.denialPromptSnippet)).some((request) => request.plannedToolName)"
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${denialRequestCursor}`)).filter((request) => String(request.allInputText ?? '').includes(config.denialPromptSnippet)).some((request) => request.plannedToolName)"
message: denied personal approval turn should not plan a tool
- set: newOutbounds
value:

6
qa/scenarios/personal/failure-recovery.yaml
View File

@@ -112,9 +112,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -165,7 +165,7 @@ flow:
expr: "`personal failure recovery reply overclaimed status: ${outbound.text}`"
- set: recoveryDebugRequests
value:
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []"
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []"
- assert:
expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
message:

6
qa/scenarios/personal/no-fake-progress.yaml
View File

@@ -107,9 +107,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -160,7 +160,7 @@ flow:
expr: "`personal no-fake-progress reply overclaimed external progress: ${outbound.text}`"
- set: progressDebugRequests
value:
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal no-fake-progress check/i.test(String(request.allInputText ?? ''))) : []"
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal no-fake-progress check/i.test(String(request.allInputText ?? ''))) : []"
- assert:
expr: "!env.mock || progressDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
message:

6
qa/scenarios/personal/redaction-no-secret-leak.yaml
View File

@@ -66,9 +66,9 @@ flow:
- set: startIndex
value:
expr: state.getSnapshot().messages.length
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -89,7 +89,7 @@ flow:
- sinceIndex:
ref: startIndex
- assert:
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'read')"
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'read')"
message: expected redaction scenario to perform a safe read in mock mode
- set: newOutbounds
value:

6
qa/scenarios/personal/share-safe-diagnostics-artifact.yaml
View File

@@ -106,9 +106,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -153,7 +153,7 @@ flow:
expr: "`share-safe diagnostics reply leaked unsafe source material: ${outbound.text}`"
- set: diagnosticDebugRequests
value:
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal share-safe diagnostics check/i.test(String(request.allInputText ?? ''))) : []"
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal share-safe diagnostics check/i.test(String(request.allInputText ?? ''))) : []"
- assert:
expr: "!env.mock || diagnosticDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2"
message:

8
qa/scenarios/personal/tool-safety-followthrough.yaml
View File

@@ -55,9 +55,9 @@ flow:
args:
- ref: env
- 60000
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: runAgentPrompt
args:
- ref: env
@@ -80,7 +80,7 @@ flow:
message:
expr: "`expected short READY pre-action reply, got: ${preActionOutbound.text}`"
- assert:
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.preActionPrompt)).some((request) => request.plannedToolName)"
expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.preActionPrompt)).some((request) => request.plannedToolName)"
message: pre-approval personal tool-safety turn should not plan a tool
- set: beforeApprovalCursor
value:
@@ -102,6 +102,6 @@ flow:
- expr: liveTurnTimeoutMs(env, 20000)
- expr: "env.providerMode === 'mock-openai' ? 100 : 250"
- assert:
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.approvalPromptSnippet)).some((request) => request.plannedToolName === 'read')"
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.approvalPromptSnippet)).some((request) => request.plannedToolName === 'read')"
message: expected safe read tool followthrough in mock mode
detailsExpr: outbound.text

6
qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml
View File

@@ -48,9 +48,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: sessionKey
value:
expr: "`agent:qa:anthropic-thinking-error:${randomUUID().slice(0, 8)}`"
@@ -83,7 +83,7 @@ flow:
then:
- set: scenarioRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))"
- assert:
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.providerVariant === 'anthropic' && request.plannedToolName === 'read')"
message: expected replay-safe read request on the Anthropic mock route

6
qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml
View File

@@ -41,9 +41,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: sessionKey
value:
expr: "`agent:qa:empty-response-recovery:${randomUUID().slice(0, 8)}`"
@@ -73,7 +73,7 @@ flow:
then:
- set: scenarioRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))"
- assert:
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
message: expected replay-safe read request in mock trace

6
qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml
View File

@@ -39,9 +39,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: sessionKey
value:
expr: "`agent:qa:empty-response-exhausted:${randomUUID().slice(0, 8)}`"
@@ -67,7 +67,7 @@ flow:
then:
- set: scenarioRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))"
- assert:
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
message: expected replay-safe read request in mock trace

8
qa/scenarios/runtime/gateway-restart-inflight-run.yaml
View File

@@ -70,9 +70,9 @@ flow:
- set: restartPatch
value:
expr: "({ gateway: { controlUi: { allowedOrigins: [`http://127.0.0.1:${64000 + Math.floor(Math.random() * 999)}`] } }, tools: { codeMode: { enabled: false } } })"
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- call: startAgentRun
saveAs: started
args:
@@ -91,7 +91,7 @@ flow:
args:
- lambda:
async: true
expr: "fetchJson(`${env.mock.baseUrl}/debug/requests`).then((requests) => requests.slice(requestCountBefore).find((request) => request.plannedToolName === 'wait' && String(request.allInputText ?? '').includes(config.interruptedMarker)))"
expr: "fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`).then((requests) => requests.find((request) => request.plannedToolName === 'wait' && String(request.allInputText ?? '').includes(config.interruptedMarker)))"
- expr: liveTurnTimeoutMs(env, 120000)
- 25
- call: waitForCondition
@@ -154,7 +154,7 @@ flow:
expr: "`automatic recovery emitted ${restartNotices.length} resend notice(s); outbound=${recentOutboundSummary(state)}`"
- set: recoveryRequests
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore) : []"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []"
- assert:
expr: "recoveryRequests.some((request) => String(request.allInputText ?? '').includes('Your previous turn was interrupted by a gateway restart'))"
message: restart recovery prompt did not reach the provider

6
qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml
View File

@@ -41,9 +41,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: sessionKey
value:
expr: "`agent:qa:reasoning-only-write:${randomUUID().slice(0, 8)}`"
@@ -78,7 +78,7 @@ flow:
then:
- set: scenarioRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))"
- assert:
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'write')"
message: expected mutating write request in mock trace

6
qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml
View File

@@ -41,9 +41,9 @@ flow:
- ref: env
- 60000
- call: reset
- set: requestCountBefore
- set: requestCursorBefore
value:
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
- set: sessionKey
value:
expr: "`agent:qa:reasoning-only-recovery:${randomUUID().slice(0, 8)}`"
@@ -73,7 +73,7 @@ flow:
then:
- set: scenarioRequests
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))"
- assert:
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
message: expected replay-safe read request in mock trace

14
scripts/mcp-code-mode-gateway-e2e.ts
View File

@@ -7,6 +7,11 @@ import process from "node:process";
import { setTimeout as setNodeTimeout, clearTimeout as clearNodeTimeout } from "node:timers";
import { pathToFileURL } from "node:url";
import { startQaMockOpenAiServer } from "../extensions/qa-lab/src/providers/mock-openai/server.js";
import {
qaMockRequestCursorUrl,
qaMockRequestsAfterUrl,
readQaMockRequestCursor,
} from "../extensions/qa-lab/src/providers/shared/debug-request-cursor.js";
import { stageQaMockAuthProfiles } from "../extensions/qa-lab/src/providers/shared/mock-auth.js";
import { buildQaGatewayConfig } from "../extensions/qa-lab/src/qa-gateway-config.js";
import { resetConfigRuntimeState } from "../src/config/config.js";
@@ -216,7 +221,9 @@ export async function main() {
openResponsesEnabled: true,
});
const beforeRequests = (await fetchJson(`${provider.baseUrl}/debug/requests`)) as unknown[];
const requestCursorBefore = readQaMockRequestCursor(
await fetchJson(qaMockRequestCursorUrl(provider.baseUrl)),
);
const response = await fetchJson(`http://127.0.0.1:${gatewayPort}/v1/responses`, {
method: "POST",
headers: {
@@ -242,12 +249,13 @@ export async function main() {
stream: false,
}),
});
const requests = (await fetchJson(`${provider.baseUrl}/debug/requests`)) as Array<{
const laneRequests = (await fetchJson(
qaMockRequestsAfterUrl(provider.baseUrl, requestCursorBefore),
)) as Array<{
raw?: string;
body?: { tools?: unknown[] };
plannedToolName?: string;
}>;
const laneRequests = requests.slice(beforeRequests.length);
const firstRequest = laneRequests[0] ?? {};
const mentions = await readSessionLogMentions(stateDir);
const plannedTools = laneRequests