diff --git a/extensions/qa-lab/src/providers/aimock/server.test.ts b/extensions/qa-lab/src/providers/aimock/server.test.ts index de13075d0d83..99da0be17718 100644 --- a/extensions/qa-lab/src/providers/aimock/server.test.ts +++ b/extensions/qa-lab/src/providers/aimock/server.test.ts @@ -102,6 +102,72 @@ describe("qa aimock server", () => { } }); + it("reads requests after a stable debug cursor", async () => { + const server = await startQaAimockServer({ + host: "127.0.0.1", + port: 0, + }); + const post = async (text: string) => { + const response = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "aimock/gpt-5.6-luna", + stream: false, + input: [makeResponsesInput(text)], + }), + }); + expect(response.status).toBe(200); + }; + try { + expect( + await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => response.json()), + ).toEqual({ cursor: 0 }); + const debugRequestLimit = 1_000; + for (let index = 0; index < debugRequestLimit; index += 1) { + await post(`aimock cursor ${index}`); + } + const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => + response.json(), + ); + expect(cursor).toEqual({ cursor: debugRequestLimit }); + await post("aimock cursor overflow"); + + const retained = (await fetch(`${server.baseUrl}/debug/requests`).then((response) => + response.json(), + )) as Array<{ prompt?: unknown }>; + expect(retained).toHaveLength(debugRequestLimit); + expect(retained[0]?.prompt).toBe("aimock cursor 1"); + expect(retained.at(-1)?.prompt).toBe("aimock cursor overflow"); + + const after = await fetch(`${server.baseUrl}/debug/requests?after=${debugRequestLimit}`); + expect(after.status).toBe(200); + const requests = (await after.json()) as Array<{ prompt?: unknown }>; + expect(requests).toHaveLength(1); + expect(requests[0]?.prompt).toBe("aimock cursor overflow"); + + const expired = await fetch(`${server.baseUrl}/debug/requests?after=0`); + expect(expired.status).toBe(409); + expect(await expired.json()).toEqual({ + error: "request cursor expired", + after: 0, + oldestCursor: 2, + latestCursor: debugRequestLimit + 1, + }); + + const futureCursor = debugRequestLimit + 2; + const future = await fetch(`${server.baseUrl}/debug/requests?after=${futureCursor}`); + expect(future.status).toBe(409); + expect(await future.json()).toEqual({ + error: "request cursor is ahead of the latest recorded request", + after: futureCursor, + latestCursor: debugRequestLimit + 1, + }); + } finally { + await server.stop(); + } + }); + it("treats OpenAI Codex model refs as OpenAI-compatible snapshots", async () => { const server = await startQaAimockServer({ host: "127.0.0.1", diff --git a/extensions/qa-lab/src/providers/aimock/server.ts b/extensions/qa-lab/src/providers/aimock/server.ts index 482513de8962..20f99f5da9d0 100644 --- a/extensions/qa-lab/src/providers/aimock/server.ts +++ b/extensions/qa-lab/src/providers/aimock/server.ts @@ -1,11 +1,13 @@ // Qa Lab plugin module implements server behavior. import type { IncomingMessage, ServerResponse } from "node:http"; import { + type Journal, LLMock, type ChatCompletionRequest, type JournalEntry, type Mountable, } from "@copilotkit/aimock"; +import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js"; import { writeJson } from "../shared/http-json.js"; type AimockRequestSnapshot = { @@ -23,6 +25,8 @@ type AimockRequestSnapshot = { toolOutputStructuredError?: true; }; +const AIMOCK_DEBUG_REQUEST_LIMIT = 1_000; + // Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts. // This mock mirrors the wire shape so delimiter drift fails through QA timeouts. const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<>>"; @@ -217,18 +221,96 @@ function toRequestSnapshots(entries: JournalEntry[]): AimockRequestSnapshot[] { return snapshots; } -function createDebugMount(mock: LLMock): Mountable { +function createDebugMount(): Mountable { + let journal: Journal | undefined; + let nextRequestCursor = 1; + const requestCursors = new Map(); + return { - async handleRequest(_req: IncomingMessage, res: ServerResponse, pathname: string) { - const entries = mock.getRequests(); + setJournal(nextJournal) { + if (journal === nextJournal) { + return; + } + if (journal) { + throw new Error("AIMock debug request cursor journal changed unexpectedly"); + } + journal = nextJournal; + const addJournalEntry = journal.add.bind(journal); + // AIMock evicts its request journal FIFO. Assign cursors at insertion time + // so the debug boundary remains monotonic after retained entries rotate. + journal.add = (entry) => { + const recorded = addJournalEntry(entry); + requestCursors.set(recorded.id, nextRequestCursor++); + if (requestCursors.size > AIMOCK_DEBUG_REQUEST_LIMIT) { + const oldestRequestId = requestCursors.keys().next().value; + if (oldestRequestId !== undefined) { + requestCursors.delete(oldestRequestId); + } + } + return recorded; + }; + }, + async handleRequest(req: IncomingMessage, res: ServerResponse, pathname: string) { + const entries = journal?.getAll() ?? []; const snapshots = toRequestSnapshots(entries); + const cursorSnapshots = entries.map((entry, index) => { + const cursor = requestCursors.get(entry.id); + if (cursor === undefined) { + throw new Error(`AIMock debug request cursor missing for ${entry.id}`); + } + const snapshot = snapshots[index]; + if (!snapshot) { + throw new Error(`AIMock debug request snapshot missing for ${entry.id}`); + } + return { cursor, snapshot }; + }); + const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (pathname === "/last-request") { const lastSnapshot = snapshots.at(-1); writeJson(res, 200, lastSnapshot ?? { ok: false, error: "no request recorded" }); return true; } + if (pathname === "/request-cursor") { + writeJson(res, 200, { cursor: nextRequestCursor - 1 }); + return true; + } if (pathname === "/requests") { - writeJson(res, 200, snapshots); + const afterText = url.searchParams.get("after"); + if (afterText === null) { + writeJson(res, 200, snapshots); + return true; + } + const after = parseQaDebugRequestCursor(afterText); + if (after === null) { + writeJson(res, 400, { error: "after must be a non-negative safe integer" }); + return true; + } + const latestCursor = nextRequestCursor - 1; + const oldestCursor = cursorSnapshots[0]?.cursor ?? nextRequestCursor; + if (after > latestCursor) { + writeJson(res, 409, { + error: "request cursor is ahead of the latest recorded request", + after, + latestCursor, + }); + return true; + } + if (after < oldestCursor - 1) { + writeJson(res, 409, { + error: "request cursor expired", + after, + oldestCursor, + latestCursor, + }); + return true; + } + writeJson( + res, + 200, + cursorSnapshots + .filter((request) => request.cursor > after) + .map((request) => request.snapshot), + ); return true; } if (pathname === "/image-generations") { @@ -252,9 +334,10 @@ export async function startQaAimockServer(params?: { host?: string; port?: numbe port: params?.port ?? 0, strict: false, logLevel: "silent", + journalMaxEntries: AIMOCK_DEBUG_REQUEST_LIMIT, }); - mock.mount("/debug", createDebugMount(mock)); + mock.mount("/debug", createDebugMount()); mock.onMessage(/.*/, { content: "AIMOCK_QA_OK" }); await mock.start(); diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 2bc483e00873..8d0176e180e2 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -232,6 +232,76 @@ function explicitSessionsSpawnPrompt(token: string) { } describe("qa mock openai server", () => { + it("keeps cursor reads correct when retained debug requests rotate", async () => { + const server = await startMockServer(); + const debugRequestLimit = 2_000; + const readCursor = async () => + requireRecord( + await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => response.json()), + "debug request cursor", + ).cursor; + + expect(await readCursor()).toBe(0); + for (let index = 0; index < debugRequestLimit; index += 1) { + await expectResponsesJson(server, { + stream: false, + model: "gpt-5.6-luna", + input: [makeUserInput(`cursor request ${index}`)], + }); + } + const cursor = await readCursor(); + expect(cursor).toBe(debugRequestLimit); + + await expectResponsesJson(server, { + stream: false, + model: "gpt-5.6-luna", + input: [makeUserInput("cursor request overflow")], + }); + + const retained = requireArray( + await fetch(`${server.baseUrl}/debug/requests`).then((response) => response.json()), + "retained debug requests", + ); + expect(retained).toHaveLength(debugRequestLimit); + expect(requireRecord(retained[0], "retained request 0").cursor).toBe(2); + expect(requireRecord(retained.at(-1), "last retained request").cursor).toBe( + debugRequestLimit + 1, + ); + + const nextRequests = requireArray( + await fetch(`${server.baseUrl}/debug/requests?after=${cursor}`).then((response) => + response.json(), + ), + "debug requests after cursor", + ); + expect(nextRequests).toHaveLength(1); + expect(String(requireRecord(nextRequests[0], "next request").prompt)).toContain("overflow"); + + const expired = await fetch(`${server.baseUrl}/debug/requests?after=0`); + expect(expired.status).toBe(409); + expect(await expired.json()).toEqual({ + error: "request cursor expired", + after: 0, + oldestCursor: 2, + latestCursor: debugRequestLimit + 1, + }); + + const futureCursor = debugRequestLimit + 2; + const future = await fetch(`${server.baseUrl}/debug/requests?after=${futureCursor}`); + expect(future.status).toBe(409); + expect(await future.json()).toEqual({ + error: "request cursor is ahead of the latest recorded request", + after: futureCursor, + latestCursor: debugRequestLimit + 1, + }); + + const invalid = await fetch(`${server.baseUrl}/debug/requests?after=1.5`); + expect(invalid.status).toBe(400); + expect(await invalid.json()).toEqual({ + error: "after must be a non-negative safe integer", + }); + }); + it("retains enough debug requests for long shared QA runs", async () => { const server = await startMockServer(); diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 0c3d11bc3fe3..b82b20689452 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -6,6 +6,7 @@ import { escapeRegExp, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utilit import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress"; import { closeQaHttpServer } from "../../bus-server.js"; import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js"; +import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js"; import { writeJson } from "../shared/http-json.js"; type ResponsesInputItem = Record; @@ -97,6 +98,7 @@ export function resolveProviderVariant(model: string | undefined): MockOpenAiPro } type MockOpenAiRequestSnapshot = { + cursor: number; raw: string; body: Record; prompt: string; @@ -113,6 +115,8 @@ type MockOpenAiRequestSnapshot = { toolOutputStructuredError?: true; }; +type MockOpenAiRequestSnapshotInput = Omit; + // Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts. // This mock mirrors the wire shape so delimiter drift fails through QA timeouts. const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<>>"; @@ -3784,6 +3788,16 @@ export async function startQaMockOpenAiServer(params?: { }; let lastRequest: MockOpenAiRequestSnapshot | null = null; const requests: MockOpenAiRequestSnapshot[] = []; + let nextRequestCursor = 1; + const recordRequest = (snapshot: MockOpenAiRequestSnapshotInput) => { + const recorded = { ...snapshot, cursor: nextRequestCursor++ }; + lastRequest = recorded; + requests.push(recorded); + if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) { + requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT); + } + return recorded; + }; const inflightRequests = new Map(); let nextInflightRequestId = 1; const imageGenerationRequests: Array> = []; @@ -3812,8 +3826,45 @@ export async function startQaMockOpenAiServer(params?: { writeJson(res, 200, lastRequest ?? { ok: false, error: "no request recorded" }); return; } + if (req.method === "GET" && url.pathname === "/debug/request-cursor") { + writeJson(res, 200, { cursor: nextRequestCursor - 1 }); + return; + } if (req.method === "GET" && url.pathname === "/debug/requests") { - writeJson(res, 200, requests); + const afterText = url.searchParams.get("after"); + if (afterText === null) { + writeJson(res, 200, requests); + return; + } + const after = parseQaDebugRequestCursor(afterText); + if (after === null) { + writeJson(res, 400, { error: "after must be a non-negative safe integer" }); + return; + } + const latestCursor = nextRequestCursor - 1; + const oldestCursor = requests[0]?.cursor ?? nextRequestCursor; + if (after > latestCursor) { + writeJson(res, 409, { + error: "request cursor is ahead of the latest recorded request", + after, + latestCursor, + }); + return; + } + if (after < oldestCursor - 1) { + writeJson(res, 409, { + error: "request cursor expired", + after, + oldestCursor, + latestCursor, + }); + return; + } + writeJson( + res, + 200, + requests.filter((request) => request.cursor > after), + ); return; } if (req.method === "GET" && url.pathname === "/debug/inflight-requests") { @@ -3897,7 +3948,7 @@ export async function startQaMockOpenAiServer(params?: { inflightRequests.delete(inflightRequestId); } const resolvedModel = typeof body.model === "string" ? body.model : ""; - lastRequest = { + recordRequest({ raw, body, prompt, @@ -3912,11 +3963,7 @@ export async function startQaMockOpenAiServer(params?: { plannedToolArgs: extractPlannedToolArgs(events), toolOutputCallId: extractToolOutputCallId(input) || undefined, ...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}), - }; - requests.push(lastRequest); - if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) { - requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT); - } + }); if (body.stream === false) { const completion = events.at(-1); if (!completion || completion.type !== "response.completed") { @@ -3959,7 +4006,7 @@ export async function startQaMockOpenAiServer(params?: { // is what lets a single parity run diff assertions across both lanes. // Reuse the normalized model so an empty-string body.model no longer // leaks through to `lastRequest.model`. - lastRequest = { + recordRequest({ raw, body: body as Record, prompt: extractLastUserText(input), @@ -3973,11 +4020,7 @@ export async function startQaMockOpenAiServer(params?: { plannedToolArgs: extractPlannedToolArgs(events), toolOutputCallId: extractToolOutputCallId(input) || undefined, ...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}), - }; - requests.push(lastRequest); - if (requests.length > MOCK_OPENAI_DEBUG_REQUEST_LIMIT) { - requests.splice(0, requests.length - MOCK_OPENAI_DEBUG_REQUEST_LIMIT); - } + }); if (body.stream === true) { writeAnthropicSse(res, streamEvents); return; diff --git a/extensions/qa-lab/src/providers/shared/debug-request-cursor.ts b/extensions/qa-lab/src/providers/shared/debug-request-cursor.ts new file mode 100644 index 000000000000..f10bf5c71070 --- /dev/null +++ b/extensions/qa-lab/src/providers/shared/debug-request-cursor.ts @@ -0,0 +1,27 @@ +// Shared by the QA mock providers and the fixtures that consume their debug logs. +export function parseQaDebugRequestCursor(value: string): number | null { + if (!/^(?:0|[1-9]\d*)$/u.test(value)) { + return null; + } + const cursor = Number(value); + return Number.isSafeInteger(cursor) ? cursor : null; +} + +export function readQaMockRequestCursor(value: unknown): number { + const cursor = + value && typeof value === "object" && !Array.isArray(value) && "cursor" in value + ? value.cursor + : undefined; + if (typeof cursor !== "number" || !Number.isSafeInteger(cursor) || cursor < 0) { + throw new Error("mock provider request cursor response was invalid"); + } + return cursor; +} + +export function qaMockRequestCursorUrl(baseUrl: string): string { + return `${baseUrl.replace(/\/+$/u, "")}/debug/request-cursor`; +} + +export function qaMockRequestsAfterUrl(baseUrl: string, cursor: number): string { + return `${baseUrl.replace(/\/+$/u, "")}/debug/requests?after=${cursor}`; +} diff --git a/extensions/qa-lab/src/runtime-tool-fixture.test.ts b/extensions/qa-lab/src/runtime-tool-fixture.test.ts index efec7bb7a826..afe4f6085016 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.test.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.test.ts @@ -118,7 +118,7 @@ async function runMockRuntimeToolFixtureWithOutputs(params: { const failureCallId = `call-${params.toolName}-failure`; const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: promptSnippet, @@ -526,7 +526,7 @@ describe("runtime tool fixture", () => { const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -577,7 +577,7 @@ describe("runtime tool fixture", () => { const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -631,7 +631,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -678,7 +678,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -733,7 +733,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -789,7 +789,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -838,7 +838,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -890,7 +890,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -942,7 +942,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -989,7 +989,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=image_generate", @@ -1036,7 +1036,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -1091,7 +1091,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -1199,7 +1199,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -1254,7 +1254,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", @@ -1309,7 +1309,7 @@ describe("runtime tool fixture", () => { }); const fetchJson = vi .fn() - .mockResolvedValueOnce([]) + .mockResolvedValueOnce({ cursor: 0 }) .mockResolvedValueOnce([ { allInputText: "target=read", diff --git a/extensions/qa-lab/src/runtime-tool-fixture.ts b/extensions/qa-lab/src/runtime-tool-fixture.ts index cb34596e8c31..e2dd28d2b59f 100644 --- a/extensions/qa-lab/src/runtime-tool-fixture.ts +++ b/extensions/qa-lab/src/runtime-tool-fixture.ts @@ -3,6 +3,11 @@ import fs from "node:fs/promises"; import path from "node:path"; import { loadTranscriptEventsSync } from "openclaw/plugin-sdk/session-store-runtime"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + qaMockRequestCursorUrl, + qaMockRequestsAfterUrl, + readQaMockRequestCursor, +} from "./providers/shared/debug-request-cursor.js"; import { type QaRuntimeToolCoverageMetadata, readRuntimeToolCoverageMetadata, @@ -464,31 +469,27 @@ function requestLinksPlannedToolOutput( function findPlannedRequest(params: { requests: readonly QaRuntimeToolFixtureRequest[]; - requestCountBefore: number; promptSnippet: string; excludedPromptSnippet?: string; toolName: string; }) { - return params.requests - .slice(params.requestCountBefore) - .find( - (request) => - requestMatchesPrompt(request, params.promptSnippet) && - (!params.excludedPromptSnippet || - !requestMatchesPrompt(request, params.excludedPromptSnippet)) && - request.plannedToolName === params.toolName, - ); + return params.requests.find( + (request) => + requestMatchesPrompt(request, params.promptSnippet) && + (!params.excludedPromptSnippet || + !requestMatchesPrompt(request, params.excludedPromptSnippet)) && + request.plannedToolName === params.toolName, + ); } function findExecutedRequest(params: { requests: readonly QaRuntimeToolFixtureRequest[]; - requestCountBefore: number; promptSnippet: string; excludedPromptSnippet?: string; toolName: string; }) { let plannedRequest: QaRuntimeToolFixtureRequest | undefined; - for (const request of params.requests.slice(params.requestCountBefore)) { + for (const request of params.requests) { if (!requestMatchesPrompt(request, params.promptSnippet)) { continue; } @@ -667,9 +668,8 @@ export async function runRuntimeToolFixture( `failure target=${toolName}`, ); const happyPathOutputRequired = readBoolean(config.happyPathOutputRequired, true); - const requestCountBefore = env.mock - ? readQaRuntimeToolFixtureRequests(await deps.fetchJson(`${env.mock.baseUrl}/debug/requests`)) - .length + const requestCursorBefore = env.mock + ? readQaMockRequestCursor(await deps.fetchJson(qaMockRequestCursorUrl(env.mock.baseUrl))) : 0; await deps.runAgentPrompt(env, { @@ -744,31 +744,27 @@ export async function runRuntimeToolFixture( } const requests = readQaRuntimeToolFixtureRequests( - await deps.fetchJson(`${env.mock.baseUrl}/debug/requests`), + await deps.fetchJson(qaMockRequestsAfterUrl(env.mock.baseUrl, requestCursorBefore)), ); const happyPlannedRequest = findPlannedRequest({ requests, - requestCountBefore, promptSnippet, excludedPromptSnippet: failurePromptSnippet, toolName, }); const happyRequest = findExecutedRequest({ requests, - requestCountBefore, promptSnippet, excludedPromptSnippet: failurePromptSnippet, toolName, }); const failurePlannedRequest = findPlannedRequest({ requests, - requestCountBefore, promptSnippet: failurePromptSnippet, toolName, }); const failureRequest = findExecutedRequest({ requests, - requestCountBefore, promptSnippet: failurePromptSnippet, toolName, }); diff --git a/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts b/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts index c32c76950c57..db97292ab4ba 100644 --- a/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts +++ b/extensions/qa-lab/src/tool-search-gateway.fixture.test.ts @@ -10,6 +10,11 @@ import { outputText, outputToolNames, } from "./fixture-utils.js"; +import { + qaMockRequestCursorUrl, + qaMockRequestsAfterUrl, + readQaMockRequestCursor, +} from "./providers/shared/debug-request-cursor.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; import { assertToolSearchLaneResults, @@ -23,6 +28,22 @@ afterEach(() => { }); describe("tool search gateway e2e fetch helper", () => { + it("builds and validates mock request cursor reads", () => { + expect(readQaMockRequestCursor({ cursor: 42 })).toBe(42); + expect(qaMockRequestCursorUrl("http://mock.test/")).toBe( + "http://mock.test/debug/request-cursor", + ); + expect(qaMockRequestsAfterUrl("http://mock.test/", 42)).toBe( + "http://mock.test/debug/requests?after=42", + ); + expect(() => readQaMockRequestCursor({ cursor: -1 })).toThrow( + "mock provider request cursor response was invalid", + ); + expect(() => readQaMockRequestCursor([])).toThrow( + "mock provider request cursor response was invalid", + ); + }); + it("rejects loose numeric env limits instead of parsing prefixes", () => { expect(() => readToolSearchGatewayFetchLimits({ @@ -222,7 +243,7 @@ describe("tool search gateway e2e lane result", () => { }); const fetchMock = vi .fn() - .mockResolvedValueOnce(jsonResponse([])) + .mockResolvedValueOnce(jsonResponse({ cursor: 0 })) .mockResolvedValueOnce(jsonResponse({ output: [], status: "completed" })) .mockResolvedValueOnce( jsonResponse([ diff --git a/extensions/qa-lab/src/tool-search-gateway.fixture.ts b/extensions/qa-lab/src/tool-search-gateway.fixture.ts index 558184b85ad3..3cb834d5fc24 100644 --- a/extensions/qa-lab/src/tool-search-gateway.fixture.ts +++ b/extensions/qa-lab/src/tool-search-gateway.fixture.ts @@ -14,6 +14,11 @@ import { subtractMentionCounts, type QaFixtureFetchJsonOptions, } from "./fixture-utils.js"; +import { + qaMockRequestCursorUrl, + qaMockRequestsAfterUrl, + readQaMockRequestCursor, +} from "./providers/shared/debug-request-cursor.js"; import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; @@ -371,7 +376,9 @@ export async function runToolSearchGatewayLane(params: { stateDir, targetTool: params.fixture.targetTool, }); - const beforeRequests = (await fetchJson(`${providerBaseUrl}/debug/requests`)) as unknown[]; + const requestCursorBefore = readQaMockRequestCursor( + await fetchJson(qaMockRequestCursorUrl(providerBaseUrl)), + ); const response = await fetchJson( `${params.env.gateway.baseUrl}/v1/responses`, { @@ -403,7 +410,9 @@ export async function runToolSearchGatewayLane(params: { }, { timeoutMs: liveTurnTimeoutMs(params.env, 30_000) }, ); - const requests = (await fetchJson(`${providerBaseUrl}/debug/requests`)) as Array<{ + const laneRequests = (await fetchJson( + qaMockRequestsAfterUrl(providerBaseUrl, requestCursorBefore), + )) as Array<{ raw?: string; body?: { tools?: unknown[] }; instructions?: string; @@ -412,7 +421,6 @@ export async function runToolSearchGatewayLane(params: { toolOutput?: string; plannedToolName?: string; }>; - const laneRequests = requests.slice(beforeRequests.length); const lastRequest = laneRequests.at(-1) ?? {}; const responseStatus = (response as { status?: unknown }).status; const mentionCountsAfter = await countToolSearchSessionLogMentions({ diff --git a/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml b/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml index dc9c7d82e965..7bc30db54d31 100644 --- a/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml +++ b/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml @@ -70,9 +70,9 @@ flow: - set: targetSessionKey value: expr: "buildAgentSessionKey({ agentId: 'qa', channel: 'qa-channel', accountId: 'default', peer: { kind: 'group', id: `group:${config.conversationId}` }, dmScope: env.cfg.session?.dmScope, identityLinks: env.cfg.session?.identityLinks })" - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: state.addInboundMessage args: - conversation: @@ -91,7 +91,7 @@ flow: - lambda: async: true params: [] - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0) : true" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0) : true" - expr: liveTurnTimeoutMs(env, 60000) - 500 - call: waitForCondition @@ -100,7 +100,7 @@ flow: - lambda: async: true params: [] - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.targetPromptSnippet) && request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker) : true" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.targetPromptSnippet) && request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker) : true" - expr: liveTurnTimeoutMs(env, 90000) - 500 - call: waitForOutboundMessage @@ -128,7 +128,7 @@ flow: expr: "`expected exactly one requester-visible A2A marker after duplicate window, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: config.conversationId })}`" - set: scenarioRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ prompt: String(request.prompt ?? '').slice(0, 220), plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 220) : null })) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).map((request) => ({ prompt: String(request.prompt ?? '').slice(0, 220), plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 220) : null })) : []" - assert: expr: "!env.mock || scenarioRequests.filter((request) => request.plannedToolName === 'sessions_send' && request.plannedToolArgs?.sessionKey === targetSessionKey && request.plannedToolArgs?.timeoutSeconds === 0).length === 1" message: diff --git a/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml b/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml index d38cdad6eb1c..5501c0f4211f 100644 --- a/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml +++ b/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml @@ -54,9 +54,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - sendInbound: conversation: id: @@ -84,7 +84,7 @@ flow: expr: "`expected exactly one fallback group reply, saw ${matchingOutbound.length}`" - set: scenarioRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" - assert: expr: "!env.mock || scenarioRequests.length > 0" message: expected mock request evidence for fallback scenario diff --git a/qa/scenarios/channels/group-visible-reply-tool.yaml b/qa/scenarios/channels/group-visible-reply-tool.yaml index ec0ff85ffa49..f2cc0dd4d130 100644 --- a/qa/scenarios/channels/group-visible-reply-tool.yaml +++ b/qa/scenarios/channels/group-visible-reply-tool.yaml @@ -47,9 +47,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - sendInbound: conversation: id: @@ -65,11 +65,11 @@ flow: - lambda: async: true params: [] - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true" - expr: liveTurnTimeoutMs(env, 180000) - set: scenarioRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" - assert: expr: "!env.mock || scenarioRequests.some((request) => request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === config.expectedMarker)" message: diff --git a/qa/scenarios/channels/message-tool-stranded-final-reply.yaml b/qa/scenarios/channels/message-tool-stranded-final-reply.yaml index 5eee100503ed..5ba324f31709 100644 --- a/qa/scenarios/channels/message-tool-stranded-final-reply.yaml +++ b/qa/scenarios/channels/message-tool-stranded-final-reply.yaml @@ -55,9 +55,9 @@ flow: - set: logCursor value: expr: markGatewayLogCursor() - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - sendInbound: conversation: id: @@ -78,7 +78,7 @@ flow: - expr: liveTurnTimeoutMs(env, 180000) - set: scenarioRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" - set: strandedRequests value: expr: "scenarioRequests.filter((request) => !String(request.allInputText ?? '').includes(config.retryPromptNeedle))" diff --git a/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml b/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml index 4302682f1eb2..967123d204af 100644 --- a/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml +++ b/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml @@ -52,9 +52,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - sendInbound: conversation: id: @@ -75,7 +75,7 @@ flow: - expr: liveTurnTimeoutMs(env, 180000) - set: scenarioRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []" - set: retryRequests value: expr: "scenarioRequests.filter((request) => String(request.allInputText ?? '').includes(config.retryPromptNeedle))" @@ -110,7 +110,7 @@ flow: - expr: liveTurnTimeoutMs(env, 8000) - set: settledRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []" - set: settledRetryRequests value: expr: "settledRequests.filter((request) => String(request.allInputText ?? '').includes(config.retryPromptNeedle))" diff --git a/qa/scenarios/channels/webchat-direct-reply-routing.yaml b/qa/scenarios/channels/webchat-direct-reply-routing.yaml index b4f6137b68ec..db50f2610311 100644 --- a/qa/scenarios/channels/webchat-direct-reply-routing.yaml +++ b/qa/scenarios/channels/webchat-direct-reply-routing.yaml @@ -56,9 +56,9 @@ flow: - set: startIndex value: expr: state.getSnapshot().messages.length - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - sendInbound: conversation: id: @@ -76,7 +76,7 @@ flow: args: - lambda: async: true - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).find((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : true" - expr: liveTurnTimeoutMs(env, 60000) - 500 - call: waitForOutboundMessage @@ -93,7 +93,7 @@ flow: catch: - set: directReplyDebugRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, allInputText: String(request.allInputText ?? '').slice(0, 400), finalText: String(request.finalText ?? '').slice(0, 200), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 200) : null })) : []" - throw: expr: "`direct reply marker missing: ${directReplyError?.message ?? directReplyError}; transcript=${formatTransportTranscript(state, { conversationId })}; requests=${JSON.stringify(directReplyDebugRequests)}`" - call: waitForCondition diff --git a/qa/scenarios/goals/goal-context-next-turn.yaml b/qa/scenarios/goals/goal-context-next-turn.yaml index 7051039e08be..f77dd08e21a3 100644 --- a/qa/scenarios/goals/goal-context-next-turn.yaml +++ b/qa/scenarios/goals/goal-context-next-turn.yaml @@ -76,9 +76,9 @@ flow: timeoutMs: expr: liveTurnTimeoutMs(env, 60000) saveAs: goalStartReply - - set: requestCountBeforeSecondTurn + - set: requestCursorBeforeSecondTurn value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - set: secondOutboundIndex value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" @@ -97,7 +97,7 @@ flow: args: - lambda: async: true - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeSecondTurn).find((request) => String(request.prompt ?? '').includes(config.secondMessage))" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeSecondTurn}`)).find((request) => String(request.prompt ?? '').includes(config.secondMessage))" - expr: liveTurnTimeoutMs(env, 60000) - 100 - waitForOutbound: diff --git a/qa/scenarios/goals/goal-context-survives-compaction.yaml b/qa/scenarios/goals/goal-context-survives-compaction.yaml index 52cc2a4e1cc8..571fcb82909c 100644 --- a/qa/scenarios/goals/goal-context-survives-compaction.yaml +++ b/qa/scenarios/goals/goal-context-survives-compaction.yaml @@ -133,9 +133,9 @@ flow: expr: "!normalizeLowercaseStringOrEmpty(compactReply.text).includes('skipped')" message: expr: "`manual compaction did not compact the seeded session: ${compactReply.text}`" - - set: requestCountBeforePostCompactionTurn + - set: requestCursorBeforePostCompactionTurn value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - set: postCompactOutboundIndex value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" @@ -154,7 +154,7 @@ flow: args: - lambda: async: true - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforePostCompactionTurn).find((request) => String(request.prompt ?? '').includes(config.postCompactionMessage))" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforePostCompactionTurn}`)).find((request) => String(request.prompt ?? '').includes(config.postCompactionMessage))" - expr: liveTurnTimeoutMs(env, 60000) - 100 - waitForOutbound: diff --git a/qa/scenarios/memory/active-memory-preprompt-recall.yaml b/qa/scenarios/memory/active-memory-preprompt-recall.yaml index a9929099ddb3..317525afd251 100644 --- a/qa/scenarios/memory/active-memory-preprompt-recall.yaml +++ b/qa/scenarios/memory/active-memory-preprompt-recall.yaml @@ -122,9 +122,9 @@ flow: expr: "String(doctorFixOutput).includes('Migrated 1 Active Memory session toggle entry')" message: expr: "`doctor --fix did not migrate the Active Memory session toggle: ${doctorFixOutput}`" - - set: requestCountBeforeBaseline + - set: requestCursorBeforeBaseline value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: baselineStartIndex value: expr: "state.getSnapshot().messages.length" @@ -155,16 +155,16 @@ flow: then: - set: baselineMockRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeBaseline)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeBaseline}`))" - set: baselineSessionStore value: expr: "await readRawQaSessionStore(env)" - assert: expr: "!Array.isArray(baselineSessionStore[baselineSessionKey]?.pluginDebugEntries) || !baselineSessionStore[baselineSessionKey].pluginDebugEntries.some((pluginEntry) => pluginEntry?.pluginId === 'active-memory')" message: baseline session unexpectedly recorded active-memory plugin activity - - set: requestCountBeforeActive + - set: requestCursorBeforeActive value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: activeStartIndex value: expr: "state.getSnapshot().messages.length" @@ -227,7 +227,7 @@ flow: then: - set: mockRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeActive)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeActive}`))" - assert: expr: "mockRequests.some((request) => request.allInputText.includes('You are a memory search agent.') && request.plannedToolName === 'memory_search')" message: expected mock Active Memory search request diff --git a/qa/scenarios/memory/dreaming-shadow-trial-report.yaml b/qa/scenarios/memory/dreaming-shadow-trial-report.yaml index 585ff9a51737..e4367ef53ddc 100644 --- a/qa/scenarios/memory/dreaming-shadow-trial-report.yaml +++ b/qa/scenarios/memory/dreaming-shadow-trial-report.yaml @@ -120,9 +120,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -172,7 +172,7 @@ flow: expr: "`shadow trial reply overclaimed promotion: ${outbound.text}`" - set: shadowTrialDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /dreaming shadow trial report check/i.test(String(request.allInputText ?? ''))) : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /dreaming shadow trial report check/i.test(String(request.allInputText ?? ''))) : []" - assert: expr: "!env.mock || shadowTrialDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2" message: diff --git a/qa/scenarios/personal/approval-denial-stop.yaml b/qa/scenarios/personal/approval-denial-stop.yaml index e69421fbfe6e..4b9a2a0f98c0 100644 --- a/qa/scenarios/personal/approval-denial-stop.yaml +++ b/qa/scenarios/personal/approval-denial-stop.yaml @@ -58,9 +58,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -79,7 +79,7 @@ flow: expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.setupMarker)" - expr: liveTurnTimeoutMs(env, 20000) - assert: - expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.setupPrompt)).some((request) => request.plannedToolName)" + expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.setupPrompt)).some((request) => request.plannedToolName)" message: setup turn should not plan a tool before any denied request - set: beforeDenialCursor value: @@ -87,9 +87,9 @@ flow: - set: beforeDenialOutboundCursor value: expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound').length" - - set: denialRequestStart + - set: denialRequestCursor value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -110,7 +110,7 @@ flow: - sinceIndex: ref: beforeDenialOutboundCursor - assert: - expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(denialRequestStart).filter((request) => String(request.allInputText ?? '').includes(config.denialPromptSnippet)).some((request) => request.plannedToolName)" + expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${denialRequestCursor}`)).filter((request) => String(request.allInputText ?? '').includes(config.denialPromptSnippet)).some((request) => request.plannedToolName)" message: denied personal approval turn should not plan a tool - set: newOutbounds value: diff --git a/qa/scenarios/personal/failure-recovery.yaml b/qa/scenarios/personal/failure-recovery.yaml index b7f45c27b1e6..0b62b1db6402 100644 --- a/qa/scenarios/personal/failure-recovery.yaml +++ b/qa/scenarios/personal/failure-recovery.yaml @@ -112,9 +112,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -165,7 +165,7 @@ flow: expr: "`personal failure recovery reply overclaimed status: ${outbound.text}`" - set: recoveryDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal failure recovery check/i.test(String(request.allInputText ?? ''))) : []" - assert: expr: "!env.mock || recoveryDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2" message: diff --git a/qa/scenarios/personal/no-fake-progress.yaml b/qa/scenarios/personal/no-fake-progress.yaml index bdd201638755..2d1e673bda5a 100644 --- a/qa/scenarios/personal/no-fake-progress.yaml +++ b/qa/scenarios/personal/no-fake-progress.yaml @@ -107,9 +107,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -160,7 +160,7 @@ flow: expr: "`personal no-fake-progress reply overclaimed external progress: ${outbound.text}`" - set: progressDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal no-fake-progress check/i.test(String(request.allInputText ?? ''))) : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal no-fake-progress check/i.test(String(request.allInputText ?? ''))) : []" - assert: expr: "!env.mock || progressDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2" message: diff --git a/qa/scenarios/personal/redaction-no-secret-leak.yaml b/qa/scenarios/personal/redaction-no-secret-leak.yaml index bd43d574ad8f..68bc59bf57f2 100644 --- a/qa/scenarios/personal/redaction-no-secret-leak.yaml +++ b/qa/scenarios/personal/redaction-no-secret-leak.yaml @@ -66,9 +66,9 @@ flow: - set: startIndex value: expr: state.getSnapshot().messages.length - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -89,7 +89,7 @@ flow: - sinceIndex: ref: startIndex - assert: - expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'read')" + expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'read')" message: expected redaction scenario to perform a safe read in mock mode - set: newOutbounds value: diff --git a/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml b/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml index 506134064654..b8b8c40f3545 100644 --- a/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml +++ b/qa/scenarios/personal/share-safe-diagnostics-artifact.yaml @@ -106,9 +106,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -153,7 +153,7 @@ flow: expr: "`share-safe diagnostics reply leaked unsafe source material: ${outbound.text}`" - set: diagnosticDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].slice(requestCountBefore).filter((request) => /personal share-safe diagnostics check/i.test(String(request.allInputText ?? ''))) : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /personal share-safe diagnostics check/i.test(String(request.allInputText ?? ''))) : []" - assert: expr: "!env.mock || diagnosticDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 2" message: diff --git a/qa/scenarios/personal/tool-safety-followthrough.yaml b/qa/scenarios/personal/tool-safety-followthrough.yaml index d0e85d2b50aa..130f887564df 100644 --- a/qa/scenarios/personal/tool-safety-followthrough.yaml +++ b/qa/scenarios/personal/tool-safety-followthrough.yaml @@ -55,9 +55,9 @@ flow: args: - ref: env - 60000 - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -80,7 +80,7 @@ flow: message: expr: "`expected short READY pre-action reply, got: ${preActionOutbound.text}`" - assert: - expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.preActionPrompt)).some((request) => request.plannedToolName)" + expr: "!env.mock || !(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.preActionPrompt)).some((request) => request.plannedToolName)" message: pre-approval personal tool-safety turn should not plan a tool - set: beforeApprovalCursor value: @@ -102,6 +102,6 @@ flow: - expr: liveTurnTimeoutMs(env, 20000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - assert: - expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore).filter((request) => String(request.allInputText ?? '').includes(config.approvalPromptSnippet)).some((request) => request.plannedToolName === 'read')" + expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.approvalPromptSnippet)).some((request) => request.plannedToolName === 'read')" message: expected safe read tool followthrough in mock mode detailsExpr: outbound.text diff --git a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml index 0d51507b7b95..2ae8c7f46606 100644 --- a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml @@ -48,9 +48,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: sessionKey value: expr: "`agent:qa:anthropic-thinking-error:${randomUUID().slice(0, 8)}`" @@ -83,7 +83,7 @@ flow: then: - set: scenarioRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))" - assert: expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.providerVariant === 'anthropic' && request.plannedToolName === 'read')" message: expected replay-safe read request on the Anthropic mock route diff --git a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml index c78a554c1eec..c61d15be6f79 100644 --- a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml @@ -41,9 +41,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: sessionKey value: expr: "`agent:qa:empty-response-recovery:${randomUUID().slice(0, 8)}`" @@ -73,7 +73,7 @@ flow: then: - set: scenarioRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))" - assert: expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')" message: expected replay-safe read request in mock trace diff --git a/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml b/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml index 5019dca9a051..a59ea7f0d22a 100644 --- a/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml +++ b/qa/scenarios/runtime/empty-response-retry-budget-exhausted.yaml @@ -39,9 +39,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: sessionKey value: expr: "`agent:qa:empty-response-exhausted:${randomUUID().slice(0, 8)}`" @@ -67,7 +67,7 @@ flow: then: - set: scenarioRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))" - assert: expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')" message: expected replay-safe read request in mock trace diff --git a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml index 046431a1a0cd..2420216e5326 100644 --- a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml +++ b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml @@ -70,9 +70,9 @@ flow: - set: restartPatch value: expr: "({ gateway: { controlUi: { allowedOrigins: [`http://127.0.0.1:${64000 + Math.floor(Math.random() * 999)}`] } }, tools: { codeMode: { enabled: false } } })" - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: startAgentRun saveAs: started args: @@ -91,7 +91,7 @@ flow: args: - lambda: async: true - expr: "fetchJson(`${env.mock.baseUrl}/debug/requests`).then((requests) => requests.slice(requestCountBefore).find((request) => request.plannedToolName === 'wait' && String(request.allInputText ?? '').includes(config.interruptedMarker)))" + expr: "fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`).then((requests) => requests.find((request) => request.plannedToolName === 'wait' && String(request.allInputText ?? '').includes(config.interruptedMarker)))" - expr: liveTurnTimeoutMs(env, 120000) - 25 - call: waitForCondition @@ -154,7 +154,7 @@ flow: expr: "`automatic recovery emitted ${restartNotices.length} resend notice(s); outbound=${recentOutboundSummary(state)}`" - set: recoveryRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore) : []" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []" - assert: expr: "recoveryRequests.some((request) => String(request.allInputText ?? '').includes('Your previous turn was interrupted by a gateway restart'))" message: restart recovery prompt did not reach the provider diff --git a/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml b/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml index 747f0fa59019..4e2525b04612 100644 --- a/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml +++ b/qa/scenarios/runtime/reasoning-only-no-auto-retry-after-write.yaml @@ -41,9 +41,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: sessionKey value: expr: "`agent:qa:reasoning-only-write:${randomUUID().slice(0, 8)}`" @@ -78,7 +78,7 @@ flow: then: - set: scenarioRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))" - assert: expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'write')" message: expected mutating write request in mock trace diff --git a/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml index 9511b1c6b31c..1a47b57a7614 100644 --- a/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/reasoning-only-recovery-replay-safe-read.yaml @@ -41,9 +41,9 @@ flow: - ref: env - 60000 - call: reset - - set: requestCountBefore + - set: requestCursorBefore value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: sessionKey value: expr: "`agent:qa:reasoning-only-recovery:${randomUUID().slice(0, 8)}`" @@ -73,7 +73,7 @@ flow: then: - set: scenarioRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))" - assert: expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')" message: expected replay-safe read request in mock trace diff --git a/scripts/mcp-code-mode-gateway-e2e.ts b/scripts/mcp-code-mode-gateway-e2e.ts index 167266f83604..a751f3f70059 100644 --- a/scripts/mcp-code-mode-gateway-e2e.ts +++ b/scripts/mcp-code-mode-gateway-e2e.ts @@ -7,6 +7,11 @@ import process from "node:process"; import { setTimeout as setNodeTimeout, clearTimeout as clearNodeTimeout } from "node:timers"; import { pathToFileURL } from "node:url"; import { startQaMockOpenAiServer } from "../extensions/qa-lab/src/providers/mock-openai/server.js"; +import { + qaMockRequestCursorUrl, + qaMockRequestsAfterUrl, + readQaMockRequestCursor, +} from "../extensions/qa-lab/src/providers/shared/debug-request-cursor.js"; import { stageQaMockAuthProfiles } from "../extensions/qa-lab/src/providers/shared/mock-auth.js"; import { buildQaGatewayConfig } from "../extensions/qa-lab/src/qa-gateway-config.js"; import { resetConfigRuntimeState } from "../src/config/config.js"; @@ -216,7 +221,9 @@ export async function main() { openResponsesEnabled: true, }); - const beforeRequests = (await fetchJson(`${provider.baseUrl}/debug/requests`)) as unknown[]; + const requestCursorBefore = readQaMockRequestCursor( + await fetchJson(qaMockRequestCursorUrl(provider.baseUrl)), + ); const response = await fetchJson(`http://127.0.0.1:${gatewayPort}/v1/responses`, { method: "POST", headers: { @@ -242,12 +249,13 @@ export async function main() { stream: false, }), }); - const requests = (await fetchJson(`${provider.baseUrl}/debug/requests`)) as Array<{ + const laneRequests = (await fetchJson( + qaMockRequestsAfterUrl(provider.baseUrl, requestCursorBefore), + )) as Array<{ raw?: string; body?: { tools?: unknown[] }; plannedToolName?: string; }>; - const laneRequests = requests.slice(beforeRequests.length); const firstRequest = laneRequests[0] ?? {}; const mentions = await readSessionLogMentions(stateDir); const plannedTools = laneRequests