fix(qa-lab): unify runner control plane (#112838)

This commit is contained in:
Dallin Romney
2026-07-27 14:05:14 +08:00
committed by GitHub
parent aa9eecab1f
commit 9310586b09
22 changed files with 2738 additions and 304 deletions

View File

@@ -110,6 +110,23 @@ the QA Lab page where an operator or automation loop can give the agent a QA
mission, observe real channel behavior, and record what worked, failed, or
stayed blocked.
The Runner's Scenarios panel can launch flow, Playwright, Vitest, and script
catalog entries together. **Profile** uses the taxonomy-owned membership plan;
checking scenarios creates an explicit override, while **Profile** in the
Scenarios panel returns to server-resolved profile membership.
Config also exposes **Provider lane**, primary and alternate models,
**Execution channel**, **Channel driver**, **Evidence mode**, **Runtime pair**,
and **Runtime-pair lane** (`core`, `extended`, or `soak`). Provider/model,
runtime, and channel-driver choices remain independent: for example, Real
frontier providers can use the Crabline channel driver, and Synthetic (mock) can
use Real channels. The server resolves taxonomy membership, provider/model
eligibility, declared `execution.channel`, runtime-pair-lane membership, and
supported execution kinds before launch. The Run panel shows the selected
execution kinds plus explicit exclusions or errors. Unknown, empty explicit,
profile-incompatible, or lane-incompatible selections fail closed instead of
being replaced by a default suite.
For faster QA Lab UI iteration without rebuilding the Docker image each time,
start the stack with a bind-mounted QA Lab bundle:

View File

@@ -0,0 +1,60 @@
import type { QaProviderMode } from "./src/providers/index.js";
import type { QaRuntimePairLane, QaScenarioExecution } from "./src/scenario-catalog.js";
import type {
QaScorecardChannelDriver,
QaScorecardEvidenceMode,
} from "./src/scorecard-taxonomy.js";
export type QaLabExecutionKind = NonNullable<QaScenarioExecution["kind"]>;
export type QaLabRunSelection = {
profile: string;
channel: string | null;
channelDriver: QaScorecardChannelDriver;
evidenceMode: QaScorecardEvidenceMode;
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode: boolean;
runtimePair: ["openclaw", "codex"] | null;
runtimePairLane: QaRuntimePairLane | null;
scenarioIds: string[] | null;
};
export type QaLabResolvedRunPlan = {
status: "ready" | "invalid";
profile: string;
explicitScenarioSelection: boolean;
selectedScenarios: Array<{
id: string;
title: string;
executionKind: QaLabExecutionKind;
declaredChannel: string | null;
effectiveChannel: string | null;
}>;
executionKinds: QaLabExecutionKind[];
exclusions: Array<{
scenarioId: string;
executionKind: QaLabExecutionKind;
reasons: string[];
}>;
errors: string[];
};
type QaLabRunArtifacts = {
outputDir: string;
evidencePath: string;
reportPath: string;
summaryPath: string;
watchUrl: string;
};
export type QaLabRunnerSnapshot = {
status: "idle" | "running" | "completed" | "failed";
selection: QaLabRunSelection;
plan: QaLabResolvedRunPlan | null;
startedAt?: string;
finishedAt?: string;
artifacts: QaLabRunArtifacts | null;
error: string | null;
};

View File

@@ -556,7 +556,7 @@ describe("qa cli runtime", () => {
expectWriteContains(stdoutWrite, "QA run profile: all; categories: 1; scenarios:");
});
it("filters QA-channel-pinned scenarios from the Crabline smoke profile", async () => {
it("filters QA-channel-pinned scenarios from an implicit Crabline smoke profile", async () => {
runQaSuite.mockImplementationOnce(async () => {
await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
return flowSuiteRuntimeResult({
@@ -568,12 +568,11 @@ describe("qa cli runtime", () => {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
profile: "smoke-ci",
scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"],
});
const suiteArgs = mockFirstObjectArg(runQaSuite);
expect(suiteArgs.channelDriver).toBe("crabline");
expect(suiteArgs.scenarioIds).toEqual(["channel-top-level-reply-shape"]);
expect(suiteArgs.scenarioIds).toContain("channel-top-level-reply-shape");
expect(suiteArgs.scenarioIds).not.toEqual(
expect.arrayContaining([
"instruction-followthrough-repo-contract",
@@ -603,6 +602,20 @@ describe("qa cli runtime", () => {
);
});
it("rejects explicit profile selections with an incompatible scenario", async () => {
await expect(
runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
profile: "smoke-ci",
scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"],
}),
).rejects.toThrow(
"qa run --qa-profile smoke-ci cannot run explicitly selected scenario(s): control-ui-qa-channel-image-roundtrip (channelDriver=qa-channel).",
);
expect(runQaSuite).not.toHaveBeenCalled();
});
it("dispatches the Matrix restart scenario through the Crabline smoke profile", async () => {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
@@ -773,15 +786,20 @@ describe("qa cli runtime", () => {
expect(runQaMultipass).not.toHaveBeenCalled();
});
it("rejects runtime-pair execution for live adapters", async () => {
await expect(
runQaSuiteCommand({
it("keeps runtime-pair execution independent from live adapters", async () => {
await runQaSuiteCommand({
channelDriver: "live",
channel: "telegram",
runtimePair: "openclaw,codex",
});
expect(runQaSuite).toHaveBeenCalledWith(
expect.objectContaining({
channelDriver: "live",
channel: "telegram",
runtimePair: "openclaw,codex",
channelId: "telegram",
runtimePair: ["openclaw", "codex"],
}),
).rejects.toThrow("--runtime-pair is not supported with a live QA adapter.");
expect(runQaSuite).not.toHaveBeenCalled();
);
});
it("loads contributed adapters without preselecting a scenario channel", async () => {

View File

@@ -46,6 +46,10 @@ import { startQaLabServer } from "./lab-server.js";
import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js";
import { runQaManualLane } from "./manual-lane.runtime.js";
import { runQaMultipass } from "./multipass.runtime.js";
import {
resolveQaRunProfileExecutionSelection,
resolveQaRunProfileMembership,
} from "./profile-planning.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
import {
QA_FRONTIER_PARITY_BASELINE_LABEL,
@@ -68,6 +72,10 @@ import {
type QaProviderMode,
type QaProviderModeInput,
} from "./run-config.js";
import {
resolveQaRuntimePairLaneScenarioIds,
resolveQaRuntimePairScenarioSupport,
} from "./runtime-pair-lane-selection.js";
import type { RuntimeId } from "./runtime-parity.js";
import {
QA_RUNTIME_PAIR_LANES,
@@ -80,7 +88,6 @@ import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js"
import {
qaScorecardChannelDriverSchema,
readQaScorecardTaxonomyReport,
type QaScorecardCategoryCoverageReport,
type QaScorecardChannelDriver,
type QaScorecardEvidenceMode,
} from "./scorecard-taxonomy.js";
@@ -311,76 +318,6 @@ function parseQaRuntimePairLaneFilters(input: string[] | undefined): QaRuntimePa
return rawValues as QaRuntimePairLane[];
}
function resolveQaRuntimePairLaneScenarioIds(params: {
channelDriver?: QaScorecardChannelDriver | null;
claudeCliAuthMode?: QaCliBackendAuthMode;
primaryModel: string;
providerMode: QaProviderMode;
scenarioIds: string[];
runtimePairLanes: readonly QaRuntimePairLane[];
runtimePair: boolean;
}): {
scenarioIds: string[];
excludedLaneScenarios: string[];
excludedNonFlowScenarios: string[];
} {
if (params.runtimePairLanes.length === 0) {
return {
scenarioIds: params.scenarioIds,
excludedLaneScenarios: [],
excludedNonFlowScenarios: [],
};
}
const laneSet = new Set(params.runtimePairLanes);
const matchingScenarios = readQaScenarioPack().scenarios.filter(
(scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane),
);
if (matchingScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`,
);
}
const compatibleScenarios = params.runtimePair
? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow")
: matchingScenarios;
const laneCompatibleScenarios = compatibleScenarios.filter((scenario) =>
scenarioMatchesQaProviderLane({
scenario,
providerMode: params.providerMode,
primaryModel: params.primaryModel,
channelDriver: params.channelDriver,
channel: scenario.execution.channel,
claudeCliAuthMode: params.claudeCliAuthMode,
}),
);
const excludedLaneScenarios = compatibleScenarios
.filter((scenario) => !laneCompatibleScenarios.includes(scenario))
.map((scenario) => scenario.id);
const excludedNonFlowScenarios = params.runtimePair
? matchingScenarios
.filter((scenario) => scenario.execution.kind !== "flow")
.map((scenario) => `${scenario.id} (${scenario.execution.kind})`)
: [];
if (compatibleScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.join(", ")}.`,
);
}
if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.join(", ")}.`,
);
}
return {
scenarioIds: uniqueStrings([
...params.scenarioIds,
...laneCompatibleScenarios.map((scenario) => scenario.id),
]),
excludedLaneScenarios,
excludedNonFlowScenarios,
};
}
function rejectNonFlowScenarioIds(params: {
option: "--runner multipass" | "--runtime-pair";
scenarioIds: readonly string[];
@@ -392,12 +329,13 @@ function rejectNonFlowScenarioIds(params: {
const scenarioById = new Map(
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
);
const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
const selectedScenarios = scenarioIds.flatMap((scenarioId) => {
const scenario = scenarioById.get(scenarioId);
return scenario && scenario.execution.kind !== "flow"
? [`${scenario.id} (${scenario.execution.kind})`]
: [];
return scenario ? [scenario] : [];
});
const nonFlowScenarios = resolveQaRuntimePairScenarioSupport(
selectedScenarios,
).excludedScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`);
if (nonFlowScenarios.length > 0) {
throw new Error(
`${params.option} requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
@@ -742,43 +680,30 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
if (!profileReport) {
throw new Error(`taxonomy.yaml does not define QA run profile ${profile}.`);
}
const categories = scorecardReport.categories.filter((category) =>
qaScorecardCategoryMatchesRunProfile(category, {
const membership = resolveQaRunProfileMembership(
{
profile,
surface: opts.surface,
category: opts.category,
}),
scenarioIds: opts.scenarioIds,
},
{ scenarios: scenarioPack.scenarios, scorecardReport },
);
const categories = membership.categories;
if (categories.length === 0) {
throw new Error(formatQaRunProfileNoMatchMessage(opts));
}
const scenarioBySourcePath = new Map(
scenarioPack.scenarios.map((scenario) => [scenario.sourcePath, scenario] as const),
);
const taxonomyScenariosForCategories = uniqueStrings(
categories.flatMap((category) => category.scenarioRefs),
)
.map((scenarioRef) => scenarioBySourcePath.get(scenarioRef))
.filter((scenario): scenario is NonNullable<typeof scenario> => scenario !== undefined);
const requestedScenarioIds = uniqueStrings(
(opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean),
);
const taxonomyScenarios =
requestedScenarioIds.length === 0
? taxonomyScenariosForCategories
: taxonomyScenariosForCategories.filter((scenario) =>
requestedScenarioIds.includes(scenario.id),
);
const taxonomyScenarios = membership.selectedScenarios;
if (requestedScenarioIds.length > 0 && taxonomyScenarios.length === 0) {
throw new Error(
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${requestedScenarioIds.join(",")}.`,
);
}
const matchedScenarioIds = new Set(taxonomyScenarios.map((scenario) => scenario.id));
const missingScenarioIds = requestedScenarioIds.filter(
(scenarioId) => !matchedScenarioIds.has(scenarioId),
);
const missingScenarioIds = membership.excludedScenarioIds;
if (missingScenarioIds.length > 0) {
throw new Error(
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${missingScenarioIds.join(",")}.`,
@@ -787,29 +712,23 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile);
const normalizedProviderMode = normalizeQaProviderMode(providerMode);
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode);
const scenarios = taxonomyScenarios.filter((scenario) => {
// qa-channel is the built-in harness channel, so another driver cannot implement it.
if (
scenario.execution.channel === "qa-channel" &&
profileReport.channelDriver !== "qa-channel"
) {
return false;
}
const channel =
profileReport.channelDriver === "qa-channel"
? "qa-channel"
: (scenario.execution.channel ??
(profileReport.channelDriver === "crabline"
? OPENCLAW_CRABLINE_DEFAULT_CHANNEL
: undefined));
return scenarioMatchesQaProviderLane({
scenario,
providerMode: normalizedProviderMode,
primaryModel,
channelDriver: profileReport.channelDriver,
channel,
});
const executionSelection = resolveQaRunProfileExecutionSelection({
scenarios: taxonomyScenarios,
providerMode: normalizedProviderMode,
primaryModel,
channelDriver: profileReport.channelDriver,
defaultChannel:
profileReport.channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined,
});
if (requestedScenarioIds.length > 0 && executionSelection.excludedScenarios.length > 0) {
const exclusions = executionSelection.excludedScenarios
.map(({ scenario, reasons }) => `${scenario.id} (${reasons.join(", ")})`)
.join(", ");
throw new Error(
`qa run --qa-profile ${profile} cannot run explicitly selected scenario(s): ${exclusions}.`,
);
}
const scenarios = executionSelection.selectedScenarios;
if (scenarios.length === 0) {
throw new Error(
`qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`,
@@ -898,25 +817,6 @@ function defaultQaRunProfileProviderMode(profile: string): QaProviderModeInput {
return profile === "smoke-ci" ? "mock-openai" : DEFAULT_QA_LIVE_PROVIDER_MODE;
}
function qaScorecardCategoryMatchesRunProfile(
category: QaScorecardCategoryCoverageReport,
opts: { profile: string; surface?: string; category?: string },
): boolean {
if (!category.profiles.includes(opts.profile)) {
return false;
}
if (opts.surface?.trim()) {
const surface = opts.surface.trim();
if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) {
return false;
}
}
if (opts.category?.trim() && category.id !== opts.category.trim()) {
return false;
}
return true;
}
function formatQaRunProfileNoMatchMessage(
opts: Pick<QaProfileCommandOptions, "profile" | "surface" | "category">,
) {
@@ -967,8 +867,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
});
const runtimePairLanes = parseQaRuntimePairLaneFilters(opts.runtimePairLane);
const runtimePairLaneSelection = resolveQaRuntimePairLaneScenarioIds({
channel: opts.channel,
channelDriver,
claudeCliAuthMode,
defaultChannel: channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined,
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
providerMode,
scenarioIds: explicitScenarioIds,
@@ -981,12 +883,12 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
}
if (runtimePairLaneSelection.excludedNonFlowScenarios.length > 0) {
process.stderr.write(
`QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.join(", ")}\n`,
`QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}\n`,
);
}
if (runtimePairLaneSelection.excludedLaneScenarios.length > 0) {
process.stderr.write(
`QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.join(", ")}\n`,
`QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}\n`,
);
}
const allowFailures = opts.allowFailures === true;
@@ -1064,9 +966,6 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
if (runner === "multipass" && liveChannelId) {
throw new Error("--channel-driver live with --channel requires --runner host.");
}
if (runtimePair && liveChannelId) {
throw new Error("--runtime-pair is not supported with a live QA adapter.");
}
if (runner === "multipass") {
rejectNonFlowScenarioIds({ option: "--runner multipass", scenarioIds });
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);

View File

@@ -75,6 +75,9 @@ describe("qa docker harness", () => {
OPENCLAW_STATE_DIR: "/tmp/openclaw/state",
});
expect(services["qa-lab"]?.volumes).toContain("./state:/opt/openclaw-scaffold:ro");
expect(services["qa-lab"]?.volumes).toContain(
`${path.relative(outputDir, "/repo/openclaw/taxonomy.yaml").split(path.sep).join("/")}:/app/taxonomy.yaml:ro`,
);
expect(compose).toContain(' - "127.0.0.1:18889:18789"');
expect(compose).toContain(' - "127.0.0.1:43124:43123"');
expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
@@ -154,7 +157,7 @@ describe("qa docker harness", () => {
expect(result.imageName).toBe("openclaw:qa-local-prebaked");
expect(calls).toEqual([
"docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
"docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=acpx qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
]);
});
@@ -177,10 +180,14 @@ describe("qa docker harness", () => {
const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
const services = parseComposeServices(compose);
expect(compose).toContain('OPENCLAW_EXTENSIONS: "acpx qa-channel qa-lab"');
expect(services["qa-mock-openai"]?.build?.context).toBe("../repo #hash");
expect(services["qa-lab"]?.volumes).toContain(
"../repo #hash/extensions/qa-lab/web/dist:/opt/openclaw-qa-lab-ui:ro",
);
expect(services["qa-lab"]?.volumes).toContain(
"../repo #hash/taxonomy.yaml:/app/taxonomy.yaml:ro",
);
expect(services["openclaw-qa-gateway"]?.volumes).toContain(
"../repo #hash:/opt/openclaw-repo:ro",
);

View File

@@ -12,6 +12,9 @@ import { buildQaGatewayConfig } from "./qa-gateway-config.js";
const QA_LAB_INTERNAL_PORT = 43123;
const QA_LAB_UI_OVERLAY_DIR = "/opt/openclaw-qa-lab-ui";
// The QA config enables ACPX. Bake the external plugin so ephemeral Gateways do
// not block startup on a network install before their health deadline.
const QA_DOCKER_PLUGIN_SELECTION = "acpx qa-channel qa-lab";
function toPosixRelative(fromDir: string, toPath: string): string {
return path.relative(fromDir, toPath).split(path.sep).join("/");
@@ -31,7 +34,7 @@ function renderImageBlock(params: {
return ` image: ${params.imageName}\n`;
}
const context = toPosixRelative(params.outputDir, params.repoRoot) || ".";
return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "qa-channel qa-lab"\n`;
return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "${QA_DOCKER_PLUGIN_SELECTION}"\n`;
}
function renderCompose(params: {
@@ -46,6 +49,10 @@ function renderCompose(params: {
}) {
const imageBlock = renderImageBlock(params);
const repoMount = toPosixRelative(params.outputDir, params.repoRoot) || ".";
const taxonomyMount = toPosixRelative(
params.outputDir,
path.join(params.repoRoot, "taxonomy.yaml"),
);
const qaLabUiMount = toPosixRelative(
params.outputDir,
path.join(params.repoRoot, "extensions", "qa-lab", "web", "dist"),
@@ -84,6 +91,7 @@ ${imageBlock} pull_policy: never
- "127.0.0.1:${params.qaLabPort}:${QA_LAB_INTERNAL_PORT}"
volumes:
- ./state:/opt/openclaw-scaffold:ro
- ${yamlDoubleQuoted(`${taxonomyMount}:/app/taxonomy.yaml:ro`)}
${params.bindUiDist ? ` - ${yamlDoubleQuoted(`${qaLabUiMount}:${QA_LAB_UI_OVERLAY_DIR}:ro`)}\n` : ""} healthcheck:
test:
- CMD
@@ -190,7 +198,7 @@ Files:
Suggested flow:
1. Build the prebaked image once:
- \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="qa-channel qa-lab" -f Dockerfile .\`
- \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="${QA_DOCKER_PLUGIN_SELECTION}" -f Dockerfile .\`
2. Start the stack:
- \`docker compose -f docker-compose.qa.yml up${params.usePrebuiltImage ? "" : " --build"} -d\`
3. Open the QA dashboard:
@@ -354,7 +362,7 @@ export async function buildQaDockerHarnessImage(
"-t",
imageName,
"--build-arg",
"OPENCLAW_EXTENSIONS=qa-channel qa-lab",
`OPENCLAW_EXTENSIONS=${QA_DOCKER_PLUGIN_SELECTION}`,
"-f",
"Dockerfile",
".",

View File

@@ -14,6 +14,22 @@ const qaChannelMock = vi.hoisted(() => ({
startAccount: vi.fn(),
}));
const suiteLaunchMock = vi.hoisted(() => ({
runQaSuite: vi.fn(),
}));
const liveTransportMock = vi.hoisted(() => ({
adapterFactories: [{ id: "live-test-factory", matches: vi.fn(), create: vi.fn() }],
listAdapterFactories: vi.fn(),
}));
vi.mock("./suite-launch.runtime.js", () => ({
runQaSuite: suiteLaunchMock.runQaSuite,
}));
vi.mock("./live-transports/cli.js", () => ({
listLiveTransportQaAdapterFactories: liveTransportMock.listAdapterFactories,
}));
vi.mock("openclaw/plugin-sdk/qa-channel", () => ({
qaChannelPlugin: {
config: {
@@ -161,6 +177,10 @@ async function startQaLabServerForTest(params?: QaLabServerStartParams) {
}
beforeEach(() => {
suiteLaunchMock.runQaSuite.mockReset();
liveTransportMock.listAdapterFactories.mockReset();
liveTransportMock.listAdapterFactories.mockReturnValue(liveTransportMock.adapterFactories);
liveTransportMock.adapterFactories[0]!.matches.mockReset();
qaChannelMock.resolveAccount.mockReset();
qaChannelMock.resolveAccount.mockImplementation((_cfg: unknown, accountId: string) => ({
accountId,
@@ -186,10 +206,10 @@ beforeEach(() => {
});
afterEach(async () => {
captureMock.reset();
while (cleanups.length > 0) {
await cleanups.pop()?.();
}
captureMock.reset();
});
function isRetryableLocalFetchError(error: unknown) {
@@ -324,6 +344,333 @@ async function createQaLabRepoRootFixture(params?: {
}
describe("qa-lab server", () => {
it("dispatches explicit mixed-kind selections through the suite planner", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
suiteLaunchMock.runQaSuite.mockResolvedValue({
executionKind: "suite",
result: {
evidencePath: "/tmp/qa-evidence.json",
outputDir: "/tmp/qa-output",
report: "# QA report\n",
reportPath: "/tmp/qa-report.md",
scenarios: [],
summaryPath: "/tmp/qa-summary.json",
},
});
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
channelDriver: "crabline",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
}),
});
expect(response.status).toBe(202);
const launch = (await response.json()) as {
plan: {
executionKinds: string[];
selectedScenarios: Array<{
id: string;
declaredChannel: string | null;
effectiveChannel: string | null;
}>;
};
};
expect(launch.plan.executionKinds).toEqual(["flow", "playwright"]);
expect(launch.plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([
"dm-chat-baseline",
"browser-talk-start-stop",
]);
expect(launch.plan.selectedScenarios).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: "dm-chat-baseline",
declaredChannel: null,
effectiveChannel: "telegram",
}),
expect.objectContaining({
id: "browser-talk-start-stop",
declaredChannel: null,
effectiveChannel: null,
}),
]),
);
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
expect.objectContaining({
alternateModel: "openai/gpt-5.6-luna",
channelDriver: "crabline",
primaryModel: "openai/gpt-5.6-luna",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
}),
);
expect(liveTransportMock.listAdapterFactories).not.toHaveBeenCalled();
await vi.waitFor(async () => {
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
runner: {
status: string;
selection: { scenarioIds: string[] };
artifacts: { watchUrl: string };
};
};
expect(bootstrap.runner.status).toBe("completed");
expect(bootstrap.runner.artifacts.watchUrl).toBe(lab.baseUrl);
expect(bootstrap.runner.selection.scenarioIds).toEqual([
"dm-chat-baseline",
"browser-talk-start-stop",
]);
});
});
it("keeps mock providers independent from real channel adapters", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
suiteLaunchMock.runQaSuite.mockResolvedValue({
executionKind: "flow",
result: {
evidencePath: "/tmp/qa-evidence.json",
outputDir: "/tmp/qa-output",
report: "# QA report\n",
reportPath: "/tmp/qa-report.md",
scenarios: [],
summaryPath: "/tmp/qa-summary.json",
watchUrl: "http://runtime-watch.invalid",
},
});
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
channelDriver: "live",
providerMode: "mock-openai",
scenarioIds: ["dm-chat-baseline"],
}),
});
expect(response.status).toBe(202);
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
expect(liveTransportMock.listAdapterFactories).toHaveBeenCalledTimes(1);
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories: liveTransportMock.adapterFactories,
channelDriver: "live",
providerMode: "mock-openai",
}),
);
await vi.waitFor(async () => {
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
runner: { status: string; artifacts: { watchUrl: string } };
};
expect(bootstrap.runner.status).toBe("completed");
expect(bootstrap.runner.artifacts.watchUrl).toBe("http://runtime-watch.invalid");
});
});
it("allows only one concurrent request to commit a resolved suite plan", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
let finishSuite: ((value: unknown) => void) | undefined;
suiteLaunchMock.runQaSuite.mockImplementation(
() =>
new Promise((resolve) => {
finishSuite = resolve;
}),
);
const request = () =>
fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline"],
}),
});
const responses = await Promise.all([request(), request()]);
expect(
responses.map((response) => response.status).toSorted((left, right) => left - right),
).toEqual([202, 409]);
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1);
finishSuite?.({
executionKind: "flow",
result: {
evidencePath: "/tmp/qa-evidence.json",
outputDir: "/tmp/qa-output",
report: "# QA report\n",
reportPath: "/tmp/qa-report.md",
scenarios: [],
summaryPath: "/tmp/qa-summary.json",
},
});
await vi.waitFor(async () => {
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
runner: { status: string };
};
expect(bootstrap.runner.status).toBe("completed");
});
});
it("rejects empty and unknown explicit selections before dispatch", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
for (const scenarioIds of [[], ["missing-scenario"]]) {
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ scenarioIds }),
});
expect(response.status).toBe(400);
}
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
});
it("returns the resolved runtime-pair-lane plan and launches it with independent live transport", async () => {
liveTransportMock.adapterFactories[0]!.matches.mockReturnValue(true);
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
suiteLaunchMock.runQaSuite.mockResolvedValue({
executionKind: "flow",
result: {
evidencePath: "/tmp/qa-evidence.json",
outputDir: "/tmp/qa-output",
report: "# QA report\n",
reportPath: "/tmp/qa-report.md",
scenarios: [],
summaryPath: "/tmp/qa-summary.json",
watchUrl: lab.baseUrl,
},
});
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
profile: "all",
channel: "telegram",
channelDriver: "live",
evidenceMode: "slim",
providerMode: "mock-openai",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "core",
}),
});
expect(response.status).toBe(202);
const payload = (await response.json()) as {
plan: {
executionKinds: string[];
exclusions: Array<{ scenarioId: string }>;
selectedScenarios: Array<{ id: string }>;
};
};
expect(payload.plan.executionKinds).toEqual(["flow"]);
expect(payload.plan.selectedScenarios.map((scenario) => scenario.id)).toContain(
"runtime-first-hour-20-turn",
);
expect(payload.plan.exclusions.map((exclusion) => exclusion.scenarioId)).toContain(
"codex-plugin-cold-install",
);
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories: liveTransportMock.adapterFactories,
channelDriver: "live",
channelId: "telegram",
evidenceMode: "slim",
providerMode: "mock-openai",
runtimePair: ["openclaw", "codex"],
}),
);
});
it("returns explicit exclusions and errors without launching unsupported execution kinds", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
profile: "all",
channelDriver: "qa-channel",
providerMode: "live-frontier",
runtimePair: ["openclaw", "codex"],
scenarioIds: ["browser-talk-start-stop"],
}),
});
expect(response.status).toBe(400);
const payload = (await response.json()) as {
error: string;
plan: {
status: string;
exclusions: Array<{ scenarioId: string; reasons: string[] }>;
errors: string[];
};
};
expect(payload.plan.status).toBe("invalid");
expect(payload.plan.exclusions).toEqual([
expect.objectContaining({
scenarioId: "browser-talk-start-stop",
reasons: ["runtimePair requires execution.kind=flow"],
}),
]);
expect(payload.error).toContain("Explicit QA scenario selection is not runnable");
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
});
it("enforces explicit execution.channel through the shared suite channel planner", async () => {
const lab = await startQaLabServerForTest();
cleanups.push(async () => {
await lab.stop();
});
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
profile: "all",
channel: "telegram",
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["matrix-room-block-streaming"],
}),
});
expect(response.status).toBe(400);
const payload = (await response.json()) as {
plan: { exclusions: Array<{ scenarioId: string; reasons: string[] }> };
};
expect(payload.plan.exclusions).toEqual(
expect.arrayContaining([
expect.objectContaining({ scenarioId: "matrix-room-block-streaming" }),
]),
);
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
});
it("cleans up capture state when embedded gateway setup fails", async () => {
qaChannelMock.resolveAccount.mockImplementationOnce(() => {
throw new Error("embedded setup failed");
@@ -396,7 +743,8 @@ describe("qa-lab server", () => {
kickoffTask: string;
scenarios: Array<{ id: string; title: string; execution?: { kind?: string } }>;
defaults: { conversationId: string; senderId: string };
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } };
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] | null } };
runnerCatalog: { channels: string[]; profiles: Array<{ id: string }> };
};
expect(bootstrap.defaults.conversationId).toBe("qa-operator");
expect(bootstrap.defaults.senderId).toBe("qa-operator");
@@ -406,13 +754,14 @@ describe("qa-lab server", () => {
expect(bootstrap.scenarios.length).toBeGreaterThanOrEqual(10);
expect(bootstrap.scenarios.map((scenario) => scenario.id)).toContain("dm-chat-baseline");
expect(bootstrap.runner.status).toBe("idle");
expect(bootstrap.runner.selection.providerMode).toBe("live-frontier");
const flowScenarioIds = bootstrap.scenarios
.filter(
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
)
.map((scenario) => scenario.id);
expect(bootstrap.runner.selection.scenarioIds).toEqual(flowScenarioIds);
expect(bootstrap.runner.selection.providerMode).toBe("mock-openai");
expect(bootstrap.runner.selection.scenarioIds).toBeNull();
expect(bootstrap.runnerCatalog.profiles.map((profile) => profile.id)).toEqual([
"smoke-ci",
"release",
"all",
]);
expect(bootstrap.runnerCatalog.channels).toContain("qa-channel");
const startupStatus = (await (
await fetchWithRetry(`${lab.baseUrl}/api/capture/startup-status`)

View File

@@ -50,12 +50,15 @@ import type {
} from "./lab-server.types.js";
import type { QaRunnerModelOption } from "./model-catalog.runtime.js";
import { createQaChannelGatewayConfig } from "./qa-channel-transport.js";
import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
import {
createIdleQaRunnerSnapshot,
createQaRunOutputDir,
normalizeQaRunSelection,
resolveQaLabRunPlan,
} from "./run-config.js";
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js";
import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js";
type QaLabBootstrapDefaults = {
@@ -300,10 +303,53 @@ export async function startQaLabServer(
let latestReport: QaLabLatestReport | null = null;
let latestScenarioRun: QaLabScenarioRun | null = null;
const scenarioCatalog = readQaBootstrapScenarioCatalog();
const scorecardReport = readQaScorecardTaxonomyReport(scenarioCatalog.scenarios);
const runnerChannels = [
...new Set(
scenarioCatalog.scenarios
.map((scenario) => scenario.execution.channel)
.filter((channel): channel is string => Boolean(channel)),
),
].toSorted();
const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget);
let runnerModelOptions: QaRunnerModelOption[] = [];
let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading";
let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios);
const resolveServerRunPlan = async (
selection: ReturnType<typeof normalizeQaRunSelection>,
adapterFactories?: readonly QaTransportAdapterFactory[],
) => {
const crabline =
selection.channelDriver === "crabline" ? await import("@openclaw/crabline") : undefined;
return resolveQaLabRunPlan({
selection,
scenarios: scenarioCatalog.scenarios,
scorecardReport,
defaultChannel:
crabline?.OPENCLAW_CRABLINE_DEFAULT_CHANNEL ??
(selection.channelDriver === "qa-channel" ? "qa-channel" : undefined),
...(crabline
? {
supportsChannel: (channel: string) => {
try {
crabline.resolveOpenClawCrablineChannelDriverSelection({ channel });
return true;
} catch {
return false;
}
},
}
: adapterFactories
? {
supportsChannel: (channel: string) =>
adapterFactories.some((factory) =>
factory.matches({ channelId: channel, driver: "live" }),
),
}
: {}),
});
};
let runnerSnapshot = createIdleQaRunnerSnapshot(scorecardReport.profiles);
runnerSnapshot.plan = await resolveServerRunPlan(runnerSnapshot.selection);
let activeSuiteRun: Promise<void> | null = null;
let controlUiProxyTarget = params?.controlUiProxyTarget?.trim()
? new URL(params.controlUiProxyTarget)
@@ -431,6 +477,8 @@ export async function startQaLabServer(
runnerCatalog: {
status: runnerModelCatalogStatus,
real: runnerModelOptions,
profiles: scorecardReport.profiles,
channels: runnerChannels,
},
});
return;
@@ -681,17 +729,52 @@ export async function startQaLabServer(
writeError(res, 409, "QA suite run already in progress");
return;
}
const selection = normalizeQaRunSelection(
await readQaJsonBody(req),
scenarioCatalog.scenarios,
);
let selection: ReturnType<typeof normalizeQaRunSelection>;
let plan: ReturnType<typeof resolveQaLabRunPlan>;
let adapterFactories: readonly QaTransportAdapterFactory[] | undefined;
try {
selection = normalizeQaRunSelection(
await readQaJsonBody(req),
scenarioCatalog.scenarios,
scorecardReport.profiles,
);
adapterFactories =
selection.channelDriver === "live"
? (await import("./live-transports/cli.js")).listLiveTransportQaAdapterFactories()
: undefined;
plan = await resolveServerRunPlan(selection, adapterFactories);
} catch (error) {
writeError(res, 400, error);
return;
}
if (plan.status === "invalid") {
writeJson(res, 400, {
error: plan.errors.join(" "),
plan,
});
return;
}
if (activeSuiteRun) {
writeError(res, 409, "QA suite run already in progress");
return;
}
state.reset();
latestReport = null;
latestScenarioRun = null;
const startedAt = new Date().toISOString();
latestScenarioRun = withQaLabRunCounts({
kind: "suite",
status: "running",
startedAt,
scenarios: plan.selectedScenarios.map((scenario) => ({
id: scenario.id,
name: scenario.title,
status: "pending",
})),
});
runnerSnapshot = {
status: "running",
selection,
plan,
startedAt,
finishedAt: undefined,
artifacts: null,
@@ -699,38 +782,86 @@ export async function startQaLabServer(
};
activeSuiteRun = (async () => {
try {
const { runQaFlowSuite } = await import("./suite.js");
const result = await runQaFlowSuite({
const [{ runQaSuite }, channelDriverSelection] = await Promise.all([
import("./suite-launch.runtime.js"),
selection.channelDriver === "crabline" && selection.channel
? import("@openclaw/crabline").then((module) =>
module.resolveOpenClawCrablineChannelDriverSelection({
channel: selection.channel!,
}),
)
: Promise.resolve(undefined),
]);
const runtimeResult = await runQaSuite({
lab: labHandle ?? undefined,
startLab: startQaLabServer,
repoRoot,
outputDir: createQaRunOutputDir(repoRoot),
channelDriver: selection.channelDriver,
...(adapterFactories ? { adapterFactories } : {}),
...(selection.channelDriver === "live" && selection.channel
? { channelId: selection.channel }
: {}),
...(channelDriverSelection ? { channelDriverSelection } : {}),
evidenceMode: selection.evidenceMode,
providerMode: selection.providerMode,
primaryModel: selection.primaryModel,
alternateModel: selection.alternateModel,
scenarioIds: selection.scenarioIds,
fastMode: selection.fastMode,
scenarioIds: plan.selectedScenarios.map((scenario) => scenario.id),
...(selection.runtimePair ? { runtimePair: selection.runtimePair } : {}),
});
const result = runtimeResult.result;
const finishedAt = new Date().toISOString();
latestReport = {
outputPath: result.reportPath,
markdown: result.report,
generatedAt: finishedAt,
};
runnerSnapshot = {
status: "completed",
selection,
plan,
startedAt,
finishedAt: new Date().toISOString(),
finishedAt,
artifacts: {
outputDir: result.outputDir,
evidencePath: result.evidencePath,
reportPath: result.reportPath,
summaryPath: result.summaryPath,
watchUrl: result.watchUrl,
watchUrl:
"watchUrl" in result && typeof result.watchUrl === "string"
? result.watchUrl
: (labHandle?.baseUrl ?? publicBaseUrl),
},
error: null,
};
} catch (error) {
const finishedAt = new Date().toISOString();
const message = formatErrorMessage(error);
latestScenarioRun = withQaLabRunCounts({
kind: "suite",
status: "completed",
startedAt,
finishedAt,
scenarios: (latestScenarioRun?.scenarios ?? []).map((scenario) =>
scenario.status === "pending" || scenario.status === "running"
? Object.assign({}, scenario, {
status: "fail" as const,
details: message,
finishedAt,
})
: scenario,
),
});
runnerSnapshot = {
status: "failed",
selection,
plan,
startedAt,
finishedAt: new Date().toISOString(),
finishedAt,
artifacts: null,
error: formatErrorMessage(error),
error: message,
};
} finally {
activeSuiteRun = null;
@@ -738,6 +869,7 @@ export async function startQaLabServer(
})();
writeJson(res, 202, {
ok: true,
plan,
runner: runnerSnapshot,
});
return;

View File

@@ -0,0 +1,146 @@
// Qa Lab plugin module owns canonical taxonomy profile membership planning.
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { QaCliBackendAuthMode } from "./gateway-child.js";
import type { QaProviderMode } from "./model-selection.js";
import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
import { describeQaProviderLaneMismatches } from "./scenario-lane.js";
import {
readQaScorecardTaxonomyReport,
type QaScorecardCategoryCoverageReport,
type QaScorecardTaxonomyReport,
type QaScorecardChannelDriver,
} from "./scorecard-taxonomy.js";
type QaRunProfileMembership = {
categories: QaScorecardCategoryCoverageReport[];
excludedScenarioIds: string[];
profile: QaScorecardTaxonomyReport["profiles"][number];
profileScenarios: QaSeedScenarioWithSource[];
selectedScenarios: QaSeedScenarioWithSource[];
};
type QaRunProfileExecutionSelection = {
excludedScenarios: Array<{ scenario: QaSeedScenarioWithSource; reasons: string[] }>;
selectedScenarios: QaSeedScenarioWithSource[];
};
function categoryMatchesRunProfile(
category: QaScorecardCategoryCoverageReport,
opts: { profile: string; surface?: string; category?: string },
): boolean {
if (!category.profiles.includes(opts.profile)) {
return false;
}
if (opts.surface?.trim()) {
const surface = opts.surface.trim();
if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) {
return false;
}
}
return !opts.category?.trim() || category.id === opts.category.trim();
}
export function resolveQaRunProfileMembership(
opts: {
profile: string;
surface?: string;
category?: string;
scenarioIds?: readonly string[];
},
source?: {
scenarios?: QaSeedScenarioWithSource[];
scorecardReport?: QaScorecardTaxonomyReport;
},
): QaRunProfileMembership {
const scenarios = source?.scenarios ?? readQaScenarioPack().scenarios;
const scorecardReport = source?.scorecardReport ?? readQaScorecardTaxonomyReport(scenarios);
const profileId = opts.profile.trim();
const profile = scorecardReport.profiles.find((entry) => entry.id === profileId);
if (!profile) {
const profileIds = scorecardReport.profiles.map((entry) => entry.id);
if (profileIds.length === 0) {
throw new Error("taxonomy.yaml does not define QA run profiles.");
}
throw new Error(
`QA run profile must be one of ${profileIds.join(", ")}, got "${opts.profile}".`,
);
}
const categories = scorecardReport.categories.filter((category) =>
categoryMatchesRunProfile(category, {
profile: profileId,
surface: opts.surface,
category: opts.category,
}),
);
const scenarioBySourcePath = new Map(
scenarios.map((scenario) => [scenario.sourcePath, scenario] as const),
);
const profileScenarios = uniqueStrings(categories.flatMap((category) => category.scenarioRefs))
.map((scenarioRef) => scenarioBySourcePath.get(scenarioRef))
.filter((scenario): scenario is QaSeedScenarioWithSource => scenario !== undefined);
const requestedScenarioIds = uniqueStrings(
(opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean),
);
if (requestedScenarioIds.length === 0) {
return {
categories,
excludedScenarioIds: [],
profile,
profileScenarios,
selectedScenarios: profileScenarios,
};
}
const requestedScenarioIdSet = new Set(requestedScenarioIds);
const selectedScenarios = profileScenarios.filter((scenario) =>
requestedScenarioIdSet.has(scenario.id),
);
const selectedScenarioIdSet = new Set(selectedScenarios.map((scenario) => scenario.id));
return {
categories,
excludedScenarioIds: requestedScenarioIds.filter(
(scenarioId) => !selectedScenarioIdSet.has(scenarioId),
),
profile,
profileScenarios,
selectedScenarios,
};
}
export function resolveQaRunProfileExecutionSelection(params: {
scenarios: readonly QaSeedScenarioWithSource[];
providerMode: QaProviderMode;
primaryModel: string;
channelDriver: QaScorecardChannelDriver;
channel?: string | null;
defaultChannel?: string;
claudeCliAuthMode?: QaCliBackendAuthMode;
}): QaRunProfileExecutionSelection {
const selectedScenarios: QaSeedScenarioWithSource[] = [];
const excludedScenarios: QaRunProfileExecutionSelection["excludedScenarios"] = [];
for (const scenario of params.scenarios) {
const reasons: string[] = [];
// qa-channel is the built-in harness channel, so another driver cannot implement it.
if (scenario.execution.channel === "qa-channel" && params.channelDriver !== "qa-channel") {
reasons.push("channelDriver=qa-channel");
}
reasons.push(
...describeQaProviderLaneMismatches({
scenario,
providerMode: params.providerMode,
primaryModel: params.primaryModel,
channelDriver: params.channelDriver,
channel:
params.channelDriver === "qa-channel"
? "qa-channel"
: (params.channel ?? scenario.execution.channel ?? params.defaultChannel),
claudeCliAuthMode: params.claudeCliAuthMode,
}),
);
if (reasons.length > 0) {
excludedScenarios.push({ scenario, reasons: uniqueStrings(reasons) });
} else {
selectedScenarios.push(scenario);
}
}
return { excludedScenarios, selectedScenarios };
}

View File

@@ -11,14 +11,29 @@ vi.mock("./model-selection.runtime.js", () => ({
defaultQaRuntimeModelForMode,
}));
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import {
resolveQaRunProfileExecutionSelection,
resolveQaRunProfileMembership,
} from "./profile-planning.js";
import {
createIdleQaRunnerSnapshot,
createQaRunOutputDir,
normalizeQaRunSelection,
resolveQaLabRunPlan,
type QaProviderModeInput,
} from "./run-config.js";
import { readQaScenarioPack } from "./scenario-catalog.js";
import {
readQaScorecardTaxonomyReport,
type QaScorecardTaxonomyReport,
} from "./scorecard-taxonomy.js";
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
const profiles: QaScorecardTaxonomyReport["profiles"] = [
{ id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] },
{ id: "release", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
{ id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
];
const scenarios = [
{
@@ -58,17 +73,25 @@ describe("qa run config", () => {
);
});
it("creates a live-by-default selection that arms flow scenarios", () => {
expect(normalizeQaRunSelection({}, scenarios)).toEqual({
providerMode: "live-frontier",
primaryModel: DEFAULT_LIVE_FRONTIER_MODEL,
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
fastMode: true,
scenarioIds: ["dm-chat-baseline", "thread-lifecycle"],
});
it("creates a canonical smoke-profile request without copying profile membership", () => {
const expected = {
profile: "smoke-ci",
channel: null,
channelDriver: "crabline",
evidenceMode: "slim",
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.6-luna",
alternateModel: "mock-openai/gpt-5.6-luna-alt",
fastMode: false,
runtimePair: null,
runtimePairLane: null,
scenarioIds: null,
} as const;
expect(normalizeQaRunSelection({}, scenarios, profiles)).toEqual(expected);
expect(normalizeQaRunSelection({ profile: null }, scenarios, profiles)).toEqual(expected);
});
it("normalizes live selections and filters unknown scenario ids", () => {
it("normalizes live selections and deduplicates scenario ids", () => {
expect(
normalizeQaRunSelection(
{
@@ -76,15 +99,45 @@ describe("qa run config", () => {
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "",
fastMode: false,
scenarioIds: ["thread-lifecycle", "missing", "thread-lifecycle"],
scenarioIds: ["thread-lifecycle", "thread-lifecycle"],
},
scenarios,
profiles,
),
).toEqual({
profile: "all",
channel: null,
channelDriver: "live",
evidenceMode: "full",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
fastMode: true,
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["thread-lifecycle"],
});
});
it("applies the canonical profile execution defaults to explicit scenario requests", () => {
expect(
normalizeQaRunSelection(
{ profile: "all", scenarioIds: ["thread-lifecycle"] },
scenarios,
profiles,
),
).toMatchObject({
profile: "all",
channelDriver: "live",
evidenceMode: "full",
scenarioIds: ["thread-lifecycle"],
});
expect(
normalizeQaRunSelection({ scenarioIds: ["thread-lifecycle"] }, scenarios, profiles),
).toMatchObject({
profile: "all",
channelDriver: "live",
evidenceMode: "full",
scenarioIds: ["thread-lifecycle"],
});
});
@@ -96,46 +149,464 @@ describe("qa run config", () => {
providerMode: "live-openai",
},
scenarios,
profiles,
),
).toThrow("unknown QA provider mode: live-openai");
});
it("falls back to all scenarios when selection would otherwise be empty", () => {
const snapshot = createIdleQaRunnerSnapshot(scenarios);
it("keeps implicit profile membership server-owned and rejects an explicit empty selection", () => {
const snapshot = createIdleQaRunnerSnapshot(profiles);
expect(snapshot.status).toBe("idle");
expect(snapshot.selection.scenarioIds).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
expect(
normalizeQaRunSelection(
{
scenarioIds: [],
},
scenarios,
).scenarioIds,
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
expect(snapshot.selection.scenarioIds).toBeNull();
expect(snapshot.plan).toBeNull();
expect(() => normalizeQaRunSelection({ scenarioIds: [] }, scenarios, profiles)).toThrow(
"scenarioIds must be a non-empty array",
);
});
it("filters non-flow scenarios from lab runner selections", () => {
it("preserves explicit non-flow scenarios for the mixed-kind suite planner", () => {
expect(
normalizeQaRunSelection(
{
scenarioIds: ["control-ui-chat-flow-playwright", "thread-lifecycle"],
},
scenarios,
profiles,
).scenarioIds,
).toEqual(["thread-lifecycle"]);
).toEqual(["control-ui-chat-flow-playwright", "thread-lifecycle"]);
});
it("fails closed on unknown explicit scenario ids", () => {
expect(() =>
normalizeQaRunSelection(
{
scenarioIds: ["thread-lifecycle", "missing"],
},
scenarios,
profiles,
),
).toThrow("unknown QA scenario id(s): missing");
});
it("fails closed on malformed explicit profiles", () => {
for (const profile of [false, ""] as const) {
expect(() =>
normalizeQaRunSelection(
{
profile,
scenarioIds: ["thread-lifecycle"],
},
scenarios,
profiles,
),
).toThrow("QA runner profile must be a non-empty string");
}
});
it("normalizes the channel driver independently from the provider lane", () => {
expect(
normalizeQaRunSelection(
{
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline"],
},
scenarios,
profiles,
),
).toMatchObject({ channelDriver: "crabline", providerMode: "live-frontier" });
expect(
normalizeQaRunSelection(
{
channelDriver: "live",
providerMode: "mock-openai",
scenarioIds: ["dm-chat-baseline"],
},
scenarios,
profiles,
),
).toMatchObject({ channelDriver: "live", providerMode: "mock-openai" });
});
it("normalizes every server-owned control-plane axis", () => {
expect(
normalizeQaRunSelection(
{
profile: "all",
channel: " Telegram ",
channelDriver: "crabline",
evidenceMode: "slim",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "extended",
scenarioIds: ["dm-chat-baseline"],
},
scenarios,
profiles,
),
).toMatchObject({
profile: "all",
channel: "telegram",
channelDriver: "crabline",
evidenceMode: "slim",
providerMode: "live-frontier",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "extended",
scenarioIds: ["dm-chat-baseline"],
});
});
it("rejects a reversed runtime pair instead of silently changing its order", () => {
expect(() =>
normalizeQaRunSelection(
{
runtimePair: ["codex", "openclaw"],
},
scenarios,
profiles,
),
).toThrow('runtimePair must be ["openclaw", "codex"]');
});
it("shares implicit profile membership and eligibility with the canonical profile planner", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const selection = normalizeQaRunSelection({}, catalog.scenarios, scorecardReport.profiles);
const membership = resolveQaRunProfileMembership(
{ profile: selection.profile },
{ scenarios: catalog.scenarios, scorecardReport },
);
const expected = resolveQaRunProfileExecutionSelection({
scenarios: membership.selectedScenarios,
providerMode: selection.providerMode,
primaryModel: selection.primaryModel,
channelDriver: selection.channelDriver,
channel: selection.channel,
});
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
expect(plan.status).toBe("ready");
expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual(
expected.selectedScenarios.map((scenario) => scenario.id),
);
});
it("resolves mixed execution kinds and reports runtime-pair-lane exclusions", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const mixedSelection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "qa-channel",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const mixedPlan = resolveQaLabRunPlan({
selection: mixedSelection,
scenarios: catalog.scenarios,
scorecardReport,
});
expect(mixedPlan.status).toBe("ready");
expect(mixedPlan.executionKinds).toEqual(["flow", "playwright"]);
const pairSelection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "qa-channel",
providerMode: "mock-openai",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "core",
},
catalog.scenarios,
scorecardReport.profiles,
);
const pairPlan = resolveQaLabRunPlan({
selection: pairSelection,
scenarios: catalog.scenarios,
scorecardReport,
});
expect(pairPlan.status).toBe("ready");
expect(pairPlan.selectedScenarios.map((scenario) => scenario.id)).toContain(
"runtime-first-hour-20-turn",
);
expect(pairPlan.exclusions).toEqual(
expect.arrayContaining([
expect.objectContaining({
scenarioId: "codex-plugin-cold-install",
reasons: ["runtimePair requires execution.kind=flow"],
}),
]),
);
});
it("validates explicit runtime-pair-lane selections without expanding them", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const selectedId = "runtime-first-hour-20-turn";
const selection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "qa-channel",
providerMode: "mock-openai",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "core",
scenarioIds: [selectedId],
},
catalog.scenarios,
scorecardReport.profiles,
);
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
expect(plan.status).toBe("ready");
expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([selectedId]);
const outsideLaneSelection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "qa-channel",
providerMode: "mock-openai",
runtimePairLane: "core",
scenarioIds: ["dm-chat-baseline"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const outsideLanePlan = resolveQaLabRunPlan({
selection: outsideLaneSelection,
scenarios: catalog.scenarios,
scorecardReport,
});
expect(outsideLanePlan.status).toBe("invalid");
expect(outsideLanePlan.selectedScenarios).toEqual([]);
expect(outsideLanePlan.exclusions).toEqual(
expect.arrayContaining([
expect.objectContaining({
scenarioId: "dm-chat-baseline",
reasons: ["runtimePairLane=core"],
}),
]),
);
});
it("does not apply live-adapter eligibility to non-flow executions", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const selection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "live",
providerMode: "live-frontier",
scenarioIds: ["browser-talk-start-stop"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const plan = resolveQaLabRunPlan({
selection,
scenarios: catalog.scenarios,
scorecardReport,
supportsChannel: () => false,
});
expect(plan.status).toBe("ready");
expect(plan.selectedScenarios).toEqual([
expect.objectContaining({
id: "browser-talk-start-stop",
effectiveChannel: null,
}),
]);
});
it("preserves canonical unresolved live dispatch and rejects unsupported driver channels", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const liveSelection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "live",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const unresolvedLivePlan = resolveQaLabRunPlan({
selection: liveSelection,
scenarios: catalog.scenarios,
scorecardReport,
supportsChannel: () => true,
});
expect(unresolvedLivePlan.status).toBe("ready");
expect(unresolvedLivePlan.selectedScenarios).toEqual([
expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: null }),
]);
const crablineSelection = normalizeQaRunSelection(
{
profile: "all",
channel: "qa-channel",
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const unsupportedCrablinePlan = resolveQaLabRunPlan({
selection: crablineSelection,
scenarios: catalog.scenarios,
scorecardReport,
defaultChannel: "telegram",
supportsChannel: (channel) => channel === "telegram",
});
expect(unsupportedCrablinePlan.status).toBe("invalid");
expect(unsupportedCrablinePlan.exclusions).toEqual(
expect.arrayContaining([
expect.objectContaining({
scenarioId: "dm-chat-baseline",
reasons: ["unsupported crabline channel=qa-channel"],
}),
]),
);
});
it("fails closed when an explicit scenario conflicts with execution.channel", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const scenario = catalog.scenarios.find(
(entry) => entry.execution.channel && entry.execution.channel !== "qa-channel",
);
expect(scenario).toBeDefined();
const selection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "qa-channel",
providerMode: "live-frontier",
scenarioIds: [scenario!.id],
},
catalog.scenarios,
scorecardReport.profiles,
);
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
expect(plan.status).toBe("invalid");
expect(plan.exclusions).toEqual(
expect.arrayContaining([expect.objectContaining({ scenarioId: scenario!.id })]),
);
expect(plan.errors.join(" ")).toContain("Explicit QA scenario selection is not runnable");
});
it("returns an invalid resolved plan for a qa-channel execution override", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const selection = normalizeQaRunSelection(
{
profile: "all",
channel: "telegram",
channelDriver: "qa-channel",
providerMode: "mock-openai",
scenarioIds: ["dm-chat-baseline"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const plan = resolveQaLabRunPlan({
selection,
scenarios: catalog.scenarios,
scorecardReport,
defaultChannel: "qa-channel",
});
expect(plan.status).toBe("invalid");
expect(plan.errors).toContain(
"An execution channel requires channelDriver=crabline or channelDriver=live.",
);
expect(plan.selectedScenarios).toEqual([
expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: "qa-channel" }),
]);
});
it("distinguishes declared constraints from the suite-resolved effective channel", () => {
const catalog = readQaScenarioPack();
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
const selection = normalizeQaRunSelection(
{
profile: "all",
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
},
catalog.scenarios,
scorecardReport.profiles,
);
const plan = resolveQaLabRunPlan({
selection,
scenarios: catalog.scenarios,
scorecardReport,
defaultChannel: "telegram",
});
expect(plan.selectedScenarios).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: "dm-chat-baseline",
declaredChannel: null,
effectiveChannel: "telegram",
}),
expect.objectContaining({
id: "browser-talk-start-stop",
declaredChannel: null,
effectiveChannel: null,
}),
]),
);
});
it("rejects malformed requests and unknown channel drivers", () => {
expect(() => normalizeQaRunSelection(null, scenarios, profiles)).toThrow(
"request must be a JSON object",
);
expect(() =>
normalizeQaRunSelection({ channelDriver: "renamed-cli-policy" }, scenarios, profiles),
).toThrow("unknown QA channel driver: renamed-cli-policy");
});
it("keeps idle snapshots on static defaults so startup does not inspect auth profiles", () => {
defaultQaRuntimeModelForMode.mockReturnValue("openai/gpt-5.6-luna");
defaultQaRuntimeModelForMode.mockClear();
const selection = createIdleQaRunnerSnapshot(scenarios).selection;
expect(selection.providerMode).toBe("live-frontier");
expect(selection.primaryModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL);
expect(selection.alternateModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL);
const selection = createIdleQaRunnerSnapshot(profiles).selection;
expect(selection.providerMode).toBe("mock-openai");
expect(selection.primaryModel).toBe("mock-openai/gpt-5.6-luna");
expect(selection.alternateModel).toBe("mock-openai/gpt-5.6-luna-alt");
expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled();
});
it("fails closed when required canonical profiles are missing", () => {
const profilesWithoutSmoke = profiles.filter((profile) => profile.id !== "smoke-ci");
const profilesWithoutAll = profiles.filter((profile) => profile.id !== "all");
expect(() => createIdleQaRunnerSnapshot(profilesWithoutSmoke)).toThrow(
"profile table does not define required profile: smoke-ci",
);
expect(() => normalizeQaRunSelection({}, scenarios, profilesWithoutSmoke)).toThrow(
"profile table does not define required profile: smoke-ci",
);
expect(() =>
normalizeQaRunSelection({ scenarioIds: ["dm-chat-baseline"] }, scenarios, profilesWithoutAll),
).toThrow("unknown QA run profile: all");
});
it("normalizes aimock selections", () => {
expect(
normalizeQaRunSelection(
@@ -146,12 +617,19 @@ describe("qa run config", () => {
scenarioIds: ["dm-chat-baseline"],
},
scenarios,
profiles,
),
).toEqual({
profile: "all",
channel: null,
channelDriver: "live",
evidenceMode: "full",
providerMode: "aimock",
primaryModel: "aimock/gpt-5.6-luna",
alternateModel: "aimock/gpt-5.6-luna-alt",
fastMode: false,
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
});
@@ -185,12 +663,18 @@ describe("qa run config", () => {
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
expect(normalizeQaRunSelection({}, scenarios)).toEqual({
expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({
profile: "release",
channel: null,
channelDriver: "live",
evidenceMode: "full",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
fastMode: true,
scenarioIds: ["dm-chat-baseline", "thread-lifecycle"],
runtimePair: null,
runtimePairLane: null,
scenarioIds: null,
});
});
});

View File

@@ -2,8 +2,18 @@
import { randomUUID } from "node:crypto";
import path from "node:path";
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import type {
QaLabExecutionKind,
QaLabResolvedRunPlan,
QaLabRunnerSnapshot,
QaLabRunSelection,
} from "../runner-contract.js";
import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js";
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
import {
resolveQaRunProfileExecutionSelection,
resolveQaRunProfileMembership,
} from "./profile-planning.js";
import {
DEFAULT_QA_LIVE_PROVIDER_MODE,
getQaProvider,
@@ -11,35 +21,26 @@ import {
normalizeQaProviderMode as normalizeQaProviderModeInput,
type QaProviderMode,
} from "./providers/index.js";
import type { QaSeedScenario } from "./scenario-catalog.js";
import {
resolveQaRuntimePairLaneScenarioIds,
resolveQaRuntimePairScenarioSupport,
} from "./runtime-pair-lane-selection.js";
import {
qaRuntimePairLaneSchema,
type QaSeedScenario,
type QaSeedScenarioWithSource,
} from "./scenario-catalog.js";
import {
qaScorecardChannelDriverSchema,
qaScorecardEvidenceModeSchema,
type QaScorecardTaxonomyReport,
} from "./scorecard-taxonomy.js";
import { resolveQaSuiteScenarioChannels } from "./suite-planning.js";
export type { QaProviderMode } from "./model-selection.js";
export type { QaProviderModeInput } from "./providers/index.js";
type QaLabRunSelection = {
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode: boolean;
scenarioIds: string[];
};
type QaLabRunArtifacts = {
outputDir: string;
evidencePath: string;
reportPath: string;
summaryPath: string;
watchUrl: string;
};
type QaLabRunnerSnapshot = {
status: "idle" | "running" | "completed" | "failed";
selection: QaLabRunSelection;
startedAt?: string;
finishedAt?: string;
artifacts: QaLabRunArtifacts | null;
error: string | null;
};
type QaLabRunProfileOption = QaScorecardTaxonomyReport["profiles"][number];
export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
return defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined);
@@ -51,26 +52,33 @@ function defaultStaticModelForMode(mode: QaProviderMode, alternate = false) {
return defaultStaticQaModelForMode(mode, alternate ? { alternate: true } : undefined);
}
function qaLabFlowScenarioIds(scenarios: QaSeedScenario[]) {
return scenarios
.filter(
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
)
.map((scenario) => scenario.id);
function requireQaRunProfile(profiles: readonly QaLabRunProfileOption[], profileId: string) {
const profile = profiles.find((entry) => entry.id === profileId);
if (!profile) {
throw new Error(`QA runner profile table does not define required profile: ${profileId}`);
}
return profile;
}
function createDefaultQaRunSelection(
scenarios: QaSeedScenario[],
profiles: readonly QaLabRunProfileOption[],
options?: { resolveDefaultModel?: QaDefaultModelResolver },
): QaLabRunSelection {
const providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE;
const profile = requireQaRunProfile(profiles, "smoke-ci");
const providerMode: QaProviderMode = "mock-openai";
const resolveDefaultModel = options?.resolveDefaultModel ?? defaultQaModelForMode;
return {
profile: profile.id,
channel: null,
channelDriver: profile.channelDriver,
evidenceMode: profile.evidenceMode,
providerMode,
primaryModel: resolveDefaultModel(providerMode),
alternateModel: resolveDefaultModel(providerMode, true),
fastMode: true,
scenarioIds: qaLabFlowScenarioIds(scenarios),
fastMode: getQaProvider(providerMode).kind === "live",
runtimePair: null,
runtimePairLane: null,
scenarioIds: null,
};
}
@@ -90,25 +98,143 @@ function normalizeModel(input: unknown, fallback: string) {
return value || fallback;
}
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]) {
const defaultScenarioIds = qaLabFlowScenarioIds(scenarios);
const availableIds = new Set(defaultScenarioIds);
const requestedIds = Array.isArray(input)
? input
.map((value) => (typeof value === "string" ? value.trim() : ""))
.filter((value) => value.length > 0)
: [];
const selectedIds = uniqueStrings(requestedIds.filter((id) => availableIds.has(id)));
return selectedIds.length > 0 ? selectedIds : defaultScenarioIds;
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null {
if (input === undefined || input === null) {
return null;
}
if (!Array.isArray(input) || input.length === 0) {
throw new Error("QA runner scenarioIds must be a non-empty array");
}
const requestedIds = input.map((value) => {
if (typeof value !== "string" || !value.trim()) {
throw new Error("QA runner scenarioIds must contain non-empty strings");
}
return value.trim();
});
const selectedIds = uniqueStrings(requestedIds);
const availableIds = new Set(scenarios.map((scenario) => scenario.id));
const unknownIds = selectedIds.filter((id) => !availableIds.has(id));
if (unknownIds.length > 0) {
throw new Error(`unknown QA scenario id(s): ${unknownIds.join(", ")}`);
}
return selectedIds;
}
function normalizeQaChannelDriver(
input: unknown,
fallback: QaLabRunSelection["channelDriver"],
): QaLabRunSelection["channelDriver"] {
if (input === undefined || input === null || input === "") {
return fallback;
}
const parsed = qaScorecardChannelDriverSchema.safeParse(input);
if (!parsed.success) {
const details = typeof input === "string" ? `: ${input}` : "";
throw new Error(`unknown QA channel driver${details}`);
}
return parsed.data;
}
function normalizeQaProfile(
input: unknown,
profiles: readonly QaLabRunProfileOption[],
fallbackProfile?: string,
) {
const fallback = fallbackProfile ?? requireQaRunProfile(profiles, "smoke-ci").id;
// Match the other optional runner controls: null means no override, while any
// concrete profile value must be a non-empty string or the request fails closed.
if (input !== undefined && input !== null && (typeof input !== "string" || !input.trim())) {
throw new Error("QA runner profile must be a non-empty string");
}
const profile = typeof input === "string" ? input.trim() : fallback;
if (!profiles.some((entry) => entry.id === profile)) {
throw new Error(
`unknown QA run profile: ${profile}; expected one of ${profiles.map((entry) => entry.id).join(", ")}`,
);
}
return profile;
}
function normalizeQaChannel(input: unknown): string | null {
if (input === undefined || input === null || input === "") {
return null;
}
if (typeof input !== "string" || !input.trim()) {
throw new Error("QA runner channel must be a non-empty string");
}
return input.trim().toLowerCase();
}
function normalizeQaEvidenceMode(
input: unknown,
fallback: QaLabRunSelection["evidenceMode"],
): QaLabRunSelection["evidenceMode"] {
if (input === undefined || input === null || input === "") {
return fallback;
}
const parsed = qaScorecardEvidenceModeSchema.safeParse(input);
if (!parsed.success) {
const details = typeof input === "string" ? `: ${input}` : "";
throw new Error(`unknown QA evidence mode${details}`);
}
return parsed.data;
}
function normalizeQaRuntimePair(input: unknown): QaLabRunSelection["runtimePair"] {
if (input === undefined || input === null) {
return null;
}
if (
!Array.isArray(input) ||
input.length !== 2 ||
!input.every((runtime) => runtime === "openclaw" || runtime === "codex")
) {
throw new Error('QA runner runtimePair must be ["openclaw", "codex"]');
}
if (input[0] === input[1]) {
throw new Error("QA runner runtimePair must compare two different runtimes");
}
if (input[0] !== "openclaw" || input[1] !== "codex") {
throw new Error('QA runner runtimePair must be ["openclaw", "codex"]');
}
return ["openclaw", "codex"];
}
function normalizeQaRuntimePairLane(input: unknown): QaLabRunSelection["runtimePairLane"] {
if (input === undefined || input === null || input === "") {
return null;
}
const parsed = qaRuntimePairLaneSchema.safeParse(input);
if (!parsed.success) {
const details = typeof input === "string" ? `: ${input}` : "";
throw new Error(`unknown QA runtime-pair lane${details}`);
}
return parsed.data;
}
export function normalizeQaRunSelection(
input: unknown,
scenarios: QaSeedScenario[],
profiles: readonly QaLabRunProfileOption[],
): QaLabRunSelection {
const payload = input && typeof input === "object" ? (input as Record<string, unknown>) : {};
const providerMode = normalizeQaProviderMode(payload.providerMode);
if (!input || typeof input !== "object" || Array.isArray(input)) {
throw new Error("QA runner request must be a JSON object");
}
const payload = input as Record<string, unknown>;
const profile = normalizeQaProfile(
payload.profile,
profiles,
Array.isArray(payload.scenarioIds) ? "all" : undefined,
);
const profileDefaults = requireQaRunProfile(profiles, profile);
const providerMode = normalizeQaProviderMode(
payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined),
);
return {
profile,
channel: normalizeQaChannel(payload.channel),
channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver),
evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode),
providerMode,
primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)),
alternateModel: normalizeModel(
@@ -116,16 +242,226 @@ export function normalizeQaRunSelection(
defaultQaModelForMode(providerMode, true),
),
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
runtimePair: normalizeQaRuntimePair(payload.runtimePair),
runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane),
scenarioIds: normalizeScenarioIds(payload.scenarioIds, scenarios),
};
}
export function createIdleQaRunnerSnapshot(scenarios: QaSeedScenario[]): QaLabRunnerSnapshot {
function effectiveChannelForScenario(params: {
scenario: QaSeedScenarioWithSource;
selection: QaLabRunSelection;
defaultChannel?: string;
}): string | null {
// Catalog parsing defaults flow executions to kind="flow"; plans never consume raw YAML.
if (params.scenario.execution.kind !== "flow") {
return null;
}
const fallbackChannel =
params.defaultChannel ?? params.selection.channel ?? params.scenario.execution.channel;
if (!fallbackChannel) {
return null;
}
return (
resolveQaSuiteScenarioChannels({
defaultChannel: fallbackChannel,
explicitChannel:
params.selection.channelDriver === "qa-channel" ? null : params.selection.channel,
scenarios: [params.scenario],
})[0] ?? null
);
}
export function resolveQaLabRunPlan(params: {
selection: QaLabRunSelection;
scenarios: QaSeedScenarioWithSource[];
scorecardReport: QaScorecardTaxonomyReport;
defaultChannel?: string;
supportsChannel?: (channel: string) => boolean;
}): QaLabResolvedRunPlan {
const { selection } = params;
const explicitScenarioSelection = selection.scenarioIds !== null;
const membership = resolveQaRunProfileMembership(
{
profile: selection.profile,
scenarioIds: selection.scenarioIds ?? undefined,
},
{ scenarios: params.scenarios, scorecardReport: params.scorecardReport },
);
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
const exclusions: QaLabResolvedRunPlan["exclusions"] = membership.excludedScenarioIds.map(
(scenarioId) => {
const scenario = scenarioById.get(scenarioId);
return {
scenarioId,
executionKind: scenario ? scenario.execution.kind : "flow",
reasons: [`not a member of profile ${selection.profile}`],
};
},
);
let laneSelection: ReturnType<typeof resolveQaRuntimePairLaneScenarioIds>;
const errors: string[] = [];
try {
laneSelection = resolveQaRuntimePairLaneScenarioIds({
channel: selection.channel,
channelDriver: selection.channelDriver,
defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined,
primaryModel: selection.primaryModel,
providerMode: selection.providerMode,
scenarioIds: selection.runtimePairLane
? []
: membership.selectedScenarios.map((scenario) => scenario.id),
scenarios: membership.profileScenarios,
runtimePairLanes: selection.runtimePairLane ? [selection.runtimePairLane] : [],
runtimePair: selection.runtimePair !== null,
});
} catch (error) {
errors.push(error instanceof Error ? error.message : String(error));
laneSelection = {
scenarioIds: [],
excludedLaneScenarios: [],
excludedNonFlowScenarios: [],
};
}
exclusions.push(
...laneSelection.excludedLaneScenarios.map((scenario) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons: ["does not match the selected provider/model/channel lane"],
})),
...laneSelection.excludedNonFlowScenarios.map((scenario) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons: ["runtimePair requires execution.kind=flow"],
})),
);
if (selection.runtimePairLane && explicitScenarioSelection) {
const laneScenarioIds = new Set(laneSelection.scenarioIds);
const alreadyExcludedIds = new Set(exclusions.map((exclusion) => exclusion.scenarioId));
exclusions.push(
...membership.selectedScenarios
.filter(
(scenario) => !laneScenarioIds.has(scenario.id) && !alreadyExcludedIds.has(scenario.id),
)
.map((scenario) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons: [`runtimePairLane=${selection.runtimePairLane}`],
})),
);
laneSelection = {
...laneSelection,
scenarioIds: membership.selectedScenarios
.filter((scenario) => laneScenarioIds.has(scenario.id))
.map((scenario) => scenario.id),
};
}
const laneScenarios = laneSelection.scenarioIds.flatMap((scenarioId) => {
const scenario = scenarioById.get(scenarioId);
return scenario ? [scenario] : [];
});
const profileExecution = resolveQaRunProfileExecutionSelection({
scenarios: laneScenarios,
providerMode: selection.providerMode,
primaryModel: selection.primaryModel,
channelDriver: selection.channelDriver,
channel: selection.channel,
defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined,
});
exclusions.push(
...profileExecution.excludedScenarios.map(({ scenario, reasons }) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons,
})),
);
const runtimePairSupport = selection.runtimePair
? resolveQaRuntimePairScenarioSupport(profileExecution.selectedScenarios)
: { selectedScenarios: profileExecution.selectedScenarios, excludedScenarios: [] };
exclusions.push(
...runtimePairSupport.excludedScenarios.map((scenario) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons: ["runtimePair requires execution.kind=flow"],
})),
);
let selectedScenarios = runtimePairSupport.selectedScenarios;
if (selection.channelDriver !== "qa-channel" && params.supportsChannel) {
const unsupportedChannelScenarios = selectedScenarios.flatMap((scenario) => {
const channel = effectiveChannelForScenario({
scenario,
selection,
defaultChannel: params.defaultChannel,
});
if (scenario.execution.kind !== "flow") {
return [];
}
return channel !== null && !params.supportsChannel?.(channel) ? [{ scenario, channel }] : [];
});
const unsupportedIds = new Set(unsupportedChannelScenarios.map(({ scenario }) => scenario.id));
exclusions.push(
...unsupportedChannelScenarios.map(({ scenario, channel }) => ({
scenarioId: scenario.id,
executionKind: scenario.execution.kind,
reasons: [`unsupported ${selection.channelDriver} channel=${channel}`],
})),
);
selectedScenarios = selectedScenarios.filter((scenario) => !unsupportedIds.has(scenario.id));
}
if (membership.categories.length === 0) {
errors.push(`QA run profile ${selection.profile} did not resolve any taxonomy categories.`);
}
if (selection.channel && selection.channelDriver === "qa-channel") {
errors.push("An execution channel requires channelDriver=crabline or channelDriver=live.");
}
const explicitScenarioIds = new Set(selection.scenarioIds ?? []);
const explicitExclusions = exclusions.filter((exclusion) =>
explicitScenarioIds.has(exclusion.scenarioId),
);
if (explicitScenarioSelection && explicitExclusions.length > 0) {
errors.push(
`Explicit QA scenario selection is not runnable: ${explicitExclusions
.map((exclusion) => `${exclusion.scenarioId} (${exclusion.reasons.join(", ")})`)
.join("; ")}.`,
);
}
if (selectedScenarios.length === 0) {
errors.push("QA run plan selected no runnable scenarios.");
}
const executionKinds = uniqueStrings(
selectedScenarios.map((scenario) => scenario.execution.kind),
) as QaLabExecutionKind[];
return {
status: errors.length > 0 ? "invalid" : "ready",
profile: selection.profile,
explicitScenarioSelection,
selectedScenarios: selectedScenarios.map((scenario) => ({
id: scenario.id,
title: scenario.title,
executionKind: scenario.execution.kind,
declaredChannel: scenario.execution.channel ?? null,
effectiveChannel: effectiveChannelForScenario({
scenario,
selection,
defaultChannel: params.defaultChannel,
}),
})),
executionKinds,
exclusions,
errors: uniqueStrings(errors),
};
}
export function createIdleQaRunnerSnapshot(
profiles: readonly QaLabRunProfileOption[],
plan: QaLabResolvedRunPlan | null = null,
): QaLabRunnerSnapshot {
return {
status: "idle",
selection: createDefaultQaRunSelection(scenarios, {
selection: createDefaultQaRunSelection(profiles, {
resolveDefaultModel: defaultStaticModelForMode,
}),
plan,
artifacts: null,
error: null,
};

View File

@@ -0,0 +1,91 @@
// Qa Lab plugin module owns canonical runtime-pair-lane scenario selection.
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { QaCliBackendAuthMode } from "./gateway-child.js";
import type { QaProviderMode } from "./model-selection.js";
import {
readQaScenarioPack,
type QaRuntimePairLane,
type QaSeedScenarioWithSource,
} from "./scenario-catalog.js";
import { scenarioMatchesQaProviderLane } from "./scenario-lane.js";
import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js";
export function resolveQaRuntimePairScenarioSupport(
scenarios: readonly QaSeedScenarioWithSource[],
) {
return {
selectedScenarios: scenarios.filter((scenario) => scenario.execution.kind === "flow"),
excludedScenarios: scenarios.filter((scenario) => scenario.execution.kind !== "flow"),
};
}
export function resolveQaRuntimePairLaneScenarioIds(params: {
channel?: string | null;
channelDriver?: QaScorecardChannelDriver | null;
claudeCliAuthMode?: QaCliBackendAuthMode;
defaultChannel?: string;
primaryModel: string;
providerMode: QaProviderMode;
scenarioIds: string[];
scenarios?: QaSeedScenarioWithSource[];
runtimePairLanes: readonly QaRuntimePairLane[];
runtimePair: boolean;
}): {
scenarioIds: string[];
excludedLaneScenarios: QaSeedScenarioWithSource[];
excludedNonFlowScenarios: QaSeedScenarioWithSource[];
} {
if (params.runtimePairLanes.length === 0) {
return {
scenarioIds: params.scenarioIds,
excludedLaneScenarios: [],
excludedNonFlowScenarios: [],
};
}
const laneSet = new Set(params.runtimePairLanes);
const matchingScenarios = (params.scenarios ?? readQaScenarioPack().scenarios).filter(
(scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane),
);
if (matchingScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`,
);
}
const compatibleScenarios = params.runtimePair
? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow")
: matchingScenarios;
const laneCompatibleScenarios = compatibleScenarios.filter((scenario) =>
scenarioMatchesQaProviderLane({
scenario,
providerMode: params.providerMode,
primaryModel: params.primaryModel,
channelDriver: params.channelDriver,
channel: params.channel ?? scenario.execution.channel ?? params.defaultChannel,
claudeCliAuthMode: params.claudeCliAuthMode,
}),
);
const excludedLaneScenarios = compatibleScenarios.filter(
(scenario) => !laneCompatibleScenarios.includes(scenario),
);
const excludedNonFlowScenarios = params.runtimePair
? matchingScenarios.filter((scenario) => scenario.execution.kind !== "flow")
: [];
if (compatibleScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}.`,
);
}
if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) {
throw new Error(
`--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}.`,
);
}
return {
scenarioIds: uniqueStrings([
...params.scenarioIds,
...laneCompatibleScenarios.map((scenario) => scenario.id),
]),
excludedLaneScenarios,
excludedNonFlowScenarios,
};
}

View File

@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { QaLabServerHandle } from "./lab-server.types.js";
import type { QaSuiteScenarioResult } from "./suite.js";
import type {
QaTestFileScenario,
@@ -542,7 +543,19 @@ describe("qa suite runtime launcher", () => {
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
const repoRoot = await makeTempRepo("qa-suite-launch-");
const setScenarioRun = vi.fn<QaLabServerHandle["setScenarioRun"]>();
const lab = {
baseUrl: "http://127.0.0.1:43124",
listenUrl: "http://127.0.0.1:43124",
runSelfCheck: vi.fn(),
setControlUi: vi.fn(),
setLatestReport: vi.fn(),
setScenarioRun,
state: {} as QaLabServerHandle["state"],
stop: vi.fn(),
} satisfies QaLabServerHandle;
const result = await runQaSuite({
lab,
repoRoot,
outputDir: ".artifacts/qa-e2e/scenario-test",
scenarioIds: ["control-ui-chat-flow-playwright"],
@@ -582,6 +595,14 @@ describe("qa suite runtime launcher", () => {
kind: scenario.execution.kind,
})),
).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]);
expect(setScenarioRun).toHaveBeenLastCalledWith(
expect.objectContaining({
status: "completed",
scenarios: [
expect.objectContaining({ id: "control-ui-chat-flow-playwright", status: "pass" }),
],
}),
);
});
it("serializes test-file runner partitions in one checkout", async () => {
@@ -699,6 +720,96 @@ describe("qa suite runtime launcher", () => {
);
});
it("aggregates mixed-kind progress through the parent lab", async () => {
const repoRoot = await makeTempRepo("qa-suite-mixed-progress-");
const scenarioRuns: Array<Parameters<QaLabServerHandle["setScenarioRun"]>[0]> = [];
const lab = {
baseUrl: "http://127.0.0.1:43124",
listenUrl: "http://127.0.0.1:43124",
runSelfCheck: vi.fn(),
setControlUi: vi.fn(),
setLatestReport: vi.fn(),
setScenarioRun: vi.fn((run) => scenarioRuns.push(run)),
state: {} as QaLabServerHandle["state"],
stop: vi.fn(),
} satisfies QaLabServerHandle;
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
if (!defaultFlowImplementation) {
throw new Error("expected default QA flow suite mock implementation");
}
runQaFlowSuite.mockImplementationOnce(async (params) => {
params?.lab?.setScenarioRun({
kind: "suite",
status: "running",
startedAt: "2026-07-23T00:00:00.000Z",
scenarios: [
{ id: "channel-chat-baseline", name: "channel-chat-baseline", status: "running" },
],
});
params?.lab?.setScenarioRun({
kind: "suite",
status: "running",
startedAt: "2026-07-23T00:00:00.000Z",
scenarios: [
{ id: "not-a-selected-scenario", name: "channel-chat-baseline", status: "fail" },
],
});
const result = await defaultFlowImplementation(params);
params?.lab?.setScenarioRun({
kind: "suite",
status: "completed",
startedAt: "2026-07-23T00:00:00.000Z",
finishedAt: "2026-07-23T00:00:01.000Z",
scenarios: [{ id: "channel-chat-baseline", name: "channel-chat-baseline", status: "pass" }],
});
return result;
});
await runQaSuite({
lab,
repoRoot,
outputDir: ".artifacts/qa-e2e/mixed-progress",
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
});
expect(
scenarioRuns.some(
(run) =>
run?.status === "running" &&
run.scenarios.some(
(scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "running",
),
),
).toBe(true);
expect(
scenarioRuns.some((run) =>
run?.scenarios.some(
(scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "fail",
),
),
).toBe(false);
expect(
scenarioRuns.some(
(run) =>
run?.status === "running" &&
run.scenarios.some(
(scenario) =>
scenario.id === "control-ui-chat-flow-playwright" && scenario.status === "running",
),
),
).toBe(true);
expect(scenarioRuns.at(-1)).toMatchObject({
status: "completed",
scenarios: [
{ id: "channel-chat-baseline", status: "pass" },
{ id: "control-ui-chat-flow-playwright", status: "pass" },
],
});
expect(lab.setLatestReport).toHaveBeenCalledWith(
expect.objectContaining({ outputPath: expect.stringMatching(/qa-suite-report\.md$/u) }),
);
});
it("keeps channel-driver unified flow partitions serial by default", async () => {
const repoRoot = await makeTempRepo("qa-suite-crabline-serial-");
await runQaSuite({
@@ -870,6 +981,17 @@ describe("qa suite runtime launcher", () => {
it("stops unified suite partitions after the first failed flow scenario", async () => {
const repoRoot = await makeTempRepo("qa-suite-fail-fast-flow-");
const scenarioRuns: Array<Parameters<QaLabServerHandle["setScenarioRun"]>[0]> = [];
const lab = {
baseUrl: "http://127.0.0.1:43124",
listenUrl: "http://127.0.0.1:43124",
runSelfCheck: vi.fn(),
setControlUi: vi.fn(),
setLatestReport: vi.fn(),
setScenarioRun: vi.fn((run) => scenarioRuns.push(run)),
state: {} as QaLabServerHandle["state"],
stop: vi.fn(),
} satisfies QaLabServerHandle;
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
if (!defaultFlowImplementation) {
throw new Error("expected default QA flow suite mock implementation");
@@ -888,6 +1010,7 @@ describe("qa suite runtime launcher", () => {
});
const result = await runQaSuite({
lab,
repoRoot,
outputDir: ".artifacts/qa-e2e/fail-fast-flow",
concurrency: 8,
@@ -928,6 +1051,15 @@ describe("qa suite runtime launcher", () => {
"docker-npm-onboard-channel-agent",
]);
expect(summary.scenarios).toMatchObject([{ name: "dm-chat-baseline", status: "fail" }]);
expect(scenarioRuns.at(-1)).toMatchObject({
status: "completed",
scenarios: [
{ id: "dm-chat-baseline", status: "fail" },
{ id: "group-visible-reply-tool", status: "pending" },
{ id: "control-ui-chat-flow-playwright", status: "pending" },
{ id: "docker-npm-onboard-channel-agent", status: "pending" },
],
});
});
it("stops pending flow and script partitions after a native scenario fails", async () => {

View File

@@ -32,6 +32,7 @@ import {
resolveQaSuiteWorkerStartStaggerMs,
scenarioRequiresIsolatedQaSuiteWorker,
} from "./suite-planning.js";
import { createQaSuiteProgressController } from "./suite-progress.js";
import {
buildQaSuiteSummaryJson,
type QaSuiteResult,
@@ -604,6 +605,14 @@ async function runUnifiedQaSuite(params: {
const startedAt = new Date();
const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd());
const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir);
const progress = params.runParams?.lab
? createQaSuiteProgressController({
lab: params.runParams.lab,
scenarios: params.plan.scenarios,
startedAt: startedAt.toISOString(),
})
: undefined;
progress?.start();
const providerMode = normalizeQaProviderMode(
params.runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE,
);
@@ -774,6 +783,13 @@ async function runUnifiedQaSuite(params: {
}
const result = await runFlowSuite({
...params.runParams,
...(progress
? {
lab: progress.createPartitionLab(
partition.scenarios.map((scenario) => scenario.id),
),
}
: {}),
outputDir: partitionName
? flowSuitePartitionOutputDir(outputDir, partitionName)
: suitePartitionOutputDir(outputDir, "flow"),
@@ -852,6 +868,11 @@ async function runUnifiedQaSuite(params: {
const testFileScenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
const testFileStartedScenarioIds: string[] = [];
for (const [kind, testFileScenarios] of scenariosByKind) {
progress?.markRunning(
(failFast ? testFileScenarios.slice(0, 1) : testFileScenarios).map(
(scenario) => scenario.id,
),
);
const result = await runQaTestFileSuiteFromRuntime({
runParams: {
...params.runParams,
@@ -869,12 +890,12 @@ async function runUnifiedQaSuite(params: {
repoRoot,
}),
);
testFileScenarioResults.push(
...result.results.map((scenarioResult) => ({
scenarioId: scenarioResult.scenario.id,
result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot),
})),
);
const scenarioResults = result.results.map((scenarioResult) => ({
scenarioId: scenarioResult.scenario.id,
result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot),
}));
testFileScenarioResults.push(...scenarioResults);
progress?.recordResults(scenarioResults);
const resultsByScenarioId = new Map(
result.results.map((scenarioResult) => [scenarioResult.scenario.id, scenarioResult]),
);
@@ -1049,7 +1070,7 @@ async function runUnifiedQaSuite(params: {
} satisfies QaSuiteScenarioResult,
];
});
return await writeUnifiedQaSuiteArtifacts({
const unifiedResult = await writeUnifiedQaSuiteArtifacts({
alternateModel,
channelDriver: params.runParams?.channelDriver,
concurrency,
@@ -1064,6 +1085,33 @@ async function runUnifiedQaSuite(params: {
scenarios,
startedAt,
});
const progressResults = params.plan.scenarios.flatMap((scenario) => {
const result = scenarioResultsById.get(scenario.id);
if (result) {
return [{ scenarioId: scenario.id, result }];
}
if (failFast && !startedScenarioIds.has(scenario.id)) {
return [];
}
return [
{
scenarioId: scenario.id,
result: {
name: scenario.title,
status: "fail" as const,
details: "suite partition returned no scenario result",
steps: [],
},
},
];
});
progress?.complete(progressResults, finishedAt.toISOString());
params.runParams?.lab?.setLatestReport({
outputPath: unifiedResult.reportPath,
markdown: unifiedResult.report,
generatedAt: finishedAt.toISOString(),
});
return unifiedResult;
}
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {

View File

@@ -0,0 +1,126 @@
import type {
QaLabScenarioOutcome,
QaLabScenarioRun,
QaLabServerHandle,
} from "./lab-server.types.js";
type QaSuiteProgressScenario = {
id: string;
title: string;
};
type QaSuiteProgressResult = {
scenarioId: string;
result: {
name: string;
status: "pass" | "fail" | "skip";
steps: QaLabScenarioOutcome["steps"];
details?: string;
};
};
function cloneOutcome(outcome: QaLabScenarioOutcome): QaLabScenarioOutcome {
return {
...outcome,
...(outcome.steps ? { steps: outcome.steps.map((step) => ({ ...step })) } : {}),
};
}
export function createQaSuiteProgressController(params: {
lab: QaLabServerHandle;
scenarios: readonly QaSuiteProgressScenario[];
startedAt: string;
}) {
const outcomes = new Map<string, QaLabScenarioOutcome>(
params.scenarios.map((scenario) => [
scenario.id,
{
id: scenario.id,
name: scenario.title,
status: "pending" as const,
},
]),
);
const emit = (status: QaLabScenarioRun["status"], finishedAt?: string) => {
params.lab.setScenarioRun({
kind: "suite",
status,
startedAt: params.startedAt,
...(finishedAt ? { finishedAt } : {}),
scenarios: params.scenarios.map((scenario) => cloneOutcome(outcomes.get(scenario.id)!)),
});
};
const updateResult = (entry: QaSuiteProgressResult, finishedAt?: string) => {
const current = outcomes.get(entry.scenarioId);
if (!current) {
return;
}
outcomes.set(entry.scenarioId, {
...current,
name: entry.result.name,
status: entry.result.status,
...(entry.result.details ? { details: entry.result.details } : {}),
...(entry.result.steps ? { steps: entry.result.steps } : {}),
...(finishedAt ? { finishedAt } : {}),
});
};
return {
start() {
emit("running");
},
markRunning(scenarioIds: readonly string[]) {
const startedAt = new Date().toISOString();
for (const scenarioId of scenarioIds) {
const current = outcomes.get(scenarioId);
if (!current || current.status !== "pending") {
continue;
}
outcomes.set(scenarioId, { ...current, status: "running", startedAt });
}
emit("running");
},
recordResults(entries: readonly QaSuiteProgressResult[]) {
const finishedAt = new Date().toISOString();
for (const entry of entries) {
updateResult(entry, finishedAt);
}
emit("running");
},
createPartitionLab(scenarioIds: readonly string[]): QaLabServerHandle {
const partitionIds = new Set(scenarioIds);
return {
...params.lab,
setScenarioRun(next) {
if (!next) {
return;
}
for (const nextOutcome of next.scenarios) {
if (!partitionIds.has(nextOutcome.id)) {
continue;
}
const current = outcomes.get(nextOutcome.id);
if (!current) {
continue;
}
outcomes.set(nextOutcome.id, {
...current,
...nextOutcome,
});
}
emit("running");
},
// Child partition reports are incomplete. The unified owner publishes one aggregate.
setLatestReport() {},
};
},
complete(entries: readonly QaSuiteProgressResult[], finishedAt: string) {
for (const entry of entries) {
updateResult(entry, finishedAt);
}
emit("completed", finishedAt);
},
};
}

View File

@@ -0,0 +1,424 @@
/* @vitest-environment jsdom */
import { readFileSync } from "node:fs";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Bootstrap, RunnerSelection } from "./ui-types.js";
const httpMock = vi.hoisted(() => {
class QaLabHttpError extends Error {
constructor(
message: string,
readonly status: number,
readonly payload: unknown,
) {
super(message);
}
}
return {
getJson: vi.fn(),
getJsonNoStore: vi.fn(),
postJson: vi.fn(),
QaLabHttpError,
};
});
vi.mock("./http.js", () => httpMock);
import { createQaLabApp } from "./app.js";
const scenarios: Bootstrap["scenarios"] = [
{
id: "dm-chat-baseline",
title: "DM baseline",
surface: "dm",
objective: "test DM",
successCriteria: ["reply"],
execution: { kind: "flow" },
},
{
id: "browser-talk-start-stop",
title: "Browser Talk start-stop",
surface: "control-ui",
objective: "test browser Talk",
successCriteria: ["playwright pass"],
execution: { kind: "playwright" },
},
];
function createBootstrap(selection: RunnerSelection): Bootstrap {
const selectedScenarioIds = selection.scenarioIds ?? scenarios.map((scenario) => scenario.id);
return {
baseUrl: "http://127.0.0.1:43124",
controlUiEmbeddedUrl: null,
controlUiUrl: null,
defaults: {
conversationId: "qa-operator",
conversationKind: "direct",
senderId: "qa-operator",
senderName: "QA Operator",
},
kickoffTask: "Run QA",
latestReport: null,
runner: {
artifacts: null,
error: null,
plan: {
errors: [],
exclusions: [],
executionKinds: ["flow", "playwright"],
explicitScenarioSelection: selection.scenarioIds !== null,
profile: selection.profile,
selectedScenarios: scenarios
.filter((scenario) => selectedScenarioIds.includes(scenario.id))
.map((scenario) => ({
declaredChannel: null,
effectiveChannel: scenario.execution?.kind === "flow" ? "qa-channel" : null,
executionKind: scenario.execution?.kind ?? "flow",
id: scenario.id,
title: scenario.title,
})),
status: "ready",
},
selection,
status: "idle",
},
runnerCatalog: {
channels: ["matrix", "telegram"],
profiles: [
{ id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] },
{ id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
],
status: "ready",
real: [
{
input: "text",
key: "openai/gpt-5.6-luna",
name: "GPT-5.6 Luna",
preferred: true,
provider: "openai",
},
],
},
scenarios,
};
}
async function mountRunner(selection: RunnerSelection) {
let bootstrap = createBootstrap(selection);
httpMock.getJson.mockImplementation(async (url: string) => {
if (url === "/api/bootstrap") {
return bootstrap;
}
if (url === "/api/state") {
return { conversations: [], events: [], messages: [], threads: [] };
}
if (url === "/api/report") {
return { report: null };
}
if (url === "/api/outcomes") {
return { run: null };
}
if (url === "/api/capture/sessions") {
return { sessions: [] };
}
if (url === "/api/capture/startup-status") {
return {
status: {
gateway: { label: "Gateway", ok: true, url: "http://127.0.0.1:18789" },
proxy: { label: "Proxy", ok: true, url: "http://127.0.0.1:7799" },
qaLab: { label: "QA Lab", ok: true, url: bootstrap.baseUrl },
},
};
}
throw new Error(`unexpected GET ${url}`);
});
httpMock.getJsonNoStore.mockResolvedValue({ version: "test" });
httpMock.postJson.mockImplementation(async (url: string, body: unknown) => {
if (url !== "/api/scenario/suite") {
throw new Error(`unexpected POST ${url}`);
}
const nextSelection = body as RunnerSelection;
bootstrap = createBootstrap(nextSelection);
return { runner: { selection: nextSelection } };
});
const root = document.createElement("div");
document.body.append(root);
await createQaLabApp(root);
return root;
}
function selectValue(root: HTMLElement, selector: string, value: string) {
const select = root.querySelector<HTMLSelectElement>(selector);
if (!select) {
throw new Error(`missing select ${selector}`);
}
select.value = value;
select.dispatchEvent(new Event("change", { bubbles: true }));
}
beforeEach(() => {
vi.useFakeTimers();
const styles = document.createElement("style");
styles.dataset.qaLabTestStyles = "true";
styles.textContent = readFileSync(
path.join(process.cwd(), "extensions/qa-lab/web/src/styles.css"),
"utf8",
);
document.head.append(styles);
httpMock.getJson.mockReset();
httpMock.getJsonNoStore.mockReset();
httpMock.postJson.mockReset();
const storage = new Map<string, string>();
vi.stubGlobal("localStorage", {
clear: () => storage.clear(),
getItem: (key: string) => storage.get(key) ?? null,
key: (index: number) => [...storage.keys()][index] ?? null,
get length() {
return storage.size;
},
removeItem: (key: string) => storage.delete(key),
setItem: (key: string, value: string) => storage.set(key, value),
});
vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
callback(0);
return 1;
});
vi.stubGlobal("matchMedia", () => ({ matches: false }));
});
afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
vi.unstubAllGlobals();
document.body.replaceChildren();
document.querySelector("style[data-qa-lab-test-styles]")?.remove();
});
describe("QA Lab runner browser interactions", () => {
it("keeps scenario rows from collapsing inside the scrolling list", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "qa-channel",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
const scroll = root.querySelector<HTMLElement>(".scenario-scroll");
const row = root.querySelector<HTMLElement>(".scenario-item");
expect(scroll).not.toBeNull();
expect(row).not.toBeNull();
expect(getComputedStyle(scroll!).overflowY).toBe("auto");
expect(getComputedStyle(row!).flexShrink).toBe("0");
});
it("submits live-provider and Crabline selections with non-flow scenarios", async () => {
const root = await mountRunner({
alternateModel: "openai/gpt-5.6-luna",
channel: null,
channelDriver: "crabline",
evidenceMode: "full",
fastMode: true,
primaryModel: "openai/gpt-5.6-luna",
profile: "all",
providerMode: "live-frontier",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
root.querySelector<HTMLButtonElement>("[data-action='select-all-scenarios']")?.click();
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
expect(httpMock.postJson).toHaveBeenCalledWith(
"/api/scenario/suite",
expect.objectContaining({
channelDriver: "crabline",
providerMode: "live-frontier",
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
}),
);
});
it("changes to real channels without changing the mock provider lane", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "qa-channel",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
selectValue(root, "#channel-driver", "live");
selectValue(root, "#execution-channel", "telegram");
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
expect(httpMock.postJson).toHaveBeenCalledWith(
"/api/scenario/suite",
expect.objectContaining({
channelDriver: "live",
channel: "telegram",
providerMode: "mock-openai",
}),
);
});
it("submits profile, evidence, runtime-pair, lane, and channel controls", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "live",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
selectValue(root, "#run-profile", "smoke-ci");
selectValue(root, "#execution-channel", "telegram");
selectValue(root, "#evidence-mode", "slim");
selectValue(root, "#runtime-pair", "openclaw,codex");
selectValue(root, "#runtime-pair-lane", "core");
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
expect(httpMock.postJson).toHaveBeenCalledWith(
"/api/scenario/suite",
expect.objectContaining({
profile: "smoke-ci",
channel: "telegram",
channelDriver: "crabline",
evidenceMode: "slim",
runtimePair: ["openclaw", "codex"],
runtimePairLane: "core",
scenarioIds: null,
}),
);
});
it("renders server-resolved exclusions and errors from a rejected launch", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "qa-channel",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
httpMock.postJson.mockRejectedValueOnce(
new httpMock.QaLabHttpError("selection rejected", 400, {
plan: {
errors: ["Explicit QA scenario selection is not runnable."],
exclusions: [
{
executionKind: "flow",
reasons: ["channel=telegram"],
scenarioId: "dm-chat-baseline",
},
],
executionKinds: [],
explicitScenarioSelection: true,
profile: "all",
selectedScenarios: [],
status: "invalid",
},
}),
);
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
await vi.waitFor(() => expect(root.textContent).toContain("1 excluded"));
expect(root.textContent).toContain("Explicit QA scenario selection is not runnable");
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => expect(root.textContent).toContain("1 excluded"));
expect(root.textContent).toContain("Explicit QA scenario selection is not runnable");
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
selectValue(root, "#evidence-mode", "slim");
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='run']")?.click();
expect(root.textContent).not.toContain("Explicit QA scenario selection is not runnable");
expect(root.textContent).not.toContain("Resolved plan:");
});
it("starts a dirty profile override without reusing the previous resolved plan", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "qa-channel",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
selectValue(root, "#run-profile", "smoke-ci");
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='scenarios']")?.click();
root
.querySelector<HTMLInputElement>("[data-scenario-toggle-id='browser-talk-start-stop']")
?.click();
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
expect(httpMock.postJson).toHaveBeenCalledWith(
"/api/scenario/suite",
expect.objectContaining({
profile: "smoke-ci",
scenarioIds: ["browser-talk-start-stop"],
}),
);
});
it("disables launch when an explicit override becomes empty", async () => {
const root = await mountRunner({
alternateModel: "mock-openai/gpt-5.6-luna-alt",
channel: null,
channelDriver: "qa-channel",
evidenceMode: "full",
fastMode: false,
primaryModel: "mock-openai/gpt-5.6-luna",
profile: "all",
providerMode: "mock-openai",
runtimePair: null,
runtimePairLane: null,
scenarioIds: ["dm-chat-baseline"],
});
root.querySelector<HTMLInputElement>("[data-scenario-toggle-id='dm-chat-baseline']")?.click();
const runButton = root.querySelector<HTMLButtonElement>("[data-action='run-suite']");
expect(runButton?.disabled).toBe(true);
expect(runButton?.textContent).toContain("Run 0 scenarios");
runButton?.click();
expect(httpMock.postJson).not.toHaveBeenCalled();
});
});

View File

@@ -2,13 +2,14 @@
import { defaultQaModelForMode, isQaFastModeEnabled } from "../../model-selection.js";
import { normalizeCaptureSavedView, normalizeCaptureSavedViews } from "./capture-saved-view.js";
import { formatErrorMessage } from "./errors.js";
import { getJson, getJsonNoStore, postJson } from "./http.js";
import { getJson, getJsonNoStore, postJson, QaLabHttpError } from "./http.js";
import { conversationSelectionKey, findConversationBySelectionKey } from "./ui-conversation-key.js";
import {
type Bootstrap,
type EvidenceEnvelope,
type OutcomesEnvelope,
type ReportEnvelope,
type RunnerResolvedPlan,
type RunnerSelection,
type Snapshot,
type TabId,
@@ -84,6 +85,14 @@ function defaultModelsForProviderMode(
};
}
function cloneRunnerSelection(selection: RunnerSelection): RunnerSelection {
return {
...selection,
runtimePair: selection.runtimePair ? [...selection.runtimePair] : null,
scenarioIds: selection.scenarioIds ? [...selection.scenarioIds] : null,
};
}
function detectTheme(): "light" | "dark" {
const stored = localStorage.getItem("qa-lab-theme");
if (stored === "light" || stored === "dark") {
@@ -207,6 +216,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
activeTab: initialUrl.pathname === "/evidence" || initialEvidencePath ? "evidence" : "chat",
runnerDraft: null,
runnerDraftDirty: false,
runnerPlanOverride: null,
composer: {
conversationKind: "direct",
conversationId: "alice",
@@ -250,6 +260,9 @@ export async function createQaLabApp(root: HTMLDivElement) {
ra: state.bootstrap?.runner.startedAt,
rf: state.bootstrap?.runner.finishedAt,
re: state.bootstrap?.runner.error,
rpo: state.runnerPlanOverride
? `${state.runnerPlanOverride.status}:${state.runnerPlanOverride.selectedScenarios.length}:${state.runnerPlanOverride.exclusions.length}:${state.runnerPlanOverride.errors.join("|")}`
: null,
ss: state.scenarioRun?.status,
sc: state.scenarioRun?.counts,
so: state.scenarioRun?.scenarios.map((o) => o.status).join(","),
@@ -341,10 +354,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
state.evidencePathDraft = bootstrap.runner.artifacts.evidencePath;
}
if (!state.runnerDraft || !state.runnerDraftDirty) {
state.runnerDraft = {
...bootstrap.runner.selection,
scenarioIds: [...bootstrap.runner.selection.scenarioIds],
};
state.runnerDraft = cloneRunnerSelection(bootstrap.runner.selection);
state.runnerDraftDirty = false;
}
if (!state.selectedConversationKey) {
@@ -497,13 +507,14 @@ export async function createQaLabApp(root: HTMLDivElement) {
function updateRunnerDraft(mutator: (draft: RunnerSelection) => RunnerSelection) {
const fallback = state.bootstrap?.runner.selection;
if (!state.runnerDraft && fallback) {
state.runnerDraft = { ...fallback, scenarioIds: [...fallback.scenarioIds] };
state.runnerDraft = cloneRunnerSelection(fallback);
}
if (!state.runnerDraft) {
return;
}
state.runnerDraft = mutator(state.runnerDraft);
state.runnerDraftDirty = true;
state.runnerPlanOverride = null;
render();
}
@@ -610,20 +621,32 @@ export async function createQaLabApp(root: HTMLDivElement) {
const result = await postJson<{ runner: { selection: RunnerSelection } }>(
"/api/scenario/suite",
{
profile: state.runnerDraft.profile,
channel: state.runnerDraft.channel,
channelDriver: state.runnerDraft.channelDriver,
evidenceMode: state.runnerDraft.evidenceMode,
providerMode: state.runnerDraft.providerMode,
primaryModel: state.runnerDraft.primaryModel,
alternateModel: state.runnerDraft.alternateModel,
fastMode: state.runnerDraft.fastMode,
runtimePair: state.runnerDraft.runtimePair,
runtimePairLane: state.runnerDraft.runtimePairLane,
scenarioIds: state.runnerDraft.scenarioIds,
},
);
state.runnerDraft = {
...result.runner.selection,
scenarioIds: [...result.runner.selection.scenarioIds],
};
state.runnerDraft = cloneRunnerSelection(result.runner.selection);
state.runnerDraftDirty = false;
state.runnerPlanOverride = null;
state.activeTab = "chat";
await refresh();
} catch (error) {
if (error instanceof QaLabHttpError) {
const plan = (error.payload as { plan?: RunnerResolvedPlan } | null)?.plan;
if (plan) {
state.runnerPlanOverride = plan;
state.sidebarPanel = "run";
}
}
state.error = formatErrorMessage(error);
render();
} finally {
@@ -904,7 +927,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
root
.querySelector<HTMLElement>("[data-action='clear-scenarios']")
?.addEventListener("click", () => {
updateRunnerDraft((d) => ({ ...d, scenarioIds: [] }));
updateRunnerDraft((d) => ({ ...d, scenarioIds: null }));
});
/* Scenario toggles */
@@ -915,7 +938,13 @@ export async function createQaLabApp(root: HTMLDivElement) {
return;
}
updateRunnerDraft((draft) => {
const selected = new Set(draft.scenarioIds);
const selected = new Set(
draft.scenarioIds ??
(!state.runnerDraftDirty
? state.bootstrap?.runner.plan?.selectedScenarios.map((scenario) => scenario.id)
: undefined) ??
[],
);
if (node.checked) {
selected.add(scenarioId);
} else {
@@ -930,6 +959,19 @@ export async function createQaLabApp(root: HTMLDivElement) {
});
/* Config form */
root.querySelector<HTMLSelectElement>("#run-profile")?.addEventListener("change", (e) => {
const profile = (e.currentTarget as HTMLSelectElement).value;
const profileDefaults = state.bootstrap?.runnerCatalog.profiles.find(
(entry) => entry.id === profile,
);
updateRunnerDraft((draft) => ({
...draft,
profile,
channelDriver: profileDefaults?.channelDriver ?? draft.channelDriver,
evidenceMode: profileDefaults?.evidenceMode ?? draft.evidenceMode,
scenarioIds: null,
}));
});
root.querySelector<HTMLSelectElement>("#provider-mode")?.addEventListener("change", (e) => {
const mode =
(e.currentTarget as HTMLSelectElement).value === "live-frontier"
@@ -941,6 +983,33 @@ export async function createQaLabApp(root: HTMLDivElement) {
...defaultModelsForProviderMode(mode, state.bootstrap),
}));
});
root.querySelector<HTMLSelectElement>("#channel-driver")?.addEventListener("change", (e) => {
const value = (e.currentTarget as HTMLSelectElement).value;
const channelDriver = value === "crabline" || value === "live" ? value : "qa-channel";
updateRunnerDraft((draft) => ({ ...draft, channelDriver }));
});
root.querySelector<HTMLSelectElement>("#execution-channel")?.addEventListener("change", (e) => {
const channel = (e.currentTarget as HTMLSelectElement).value.trim() || null;
updateRunnerDraft((draft) => ({ ...draft, channel }));
});
root.querySelector<HTMLSelectElement>("#evidence-mode")?.addEventListener("change", (e) => {
const evidenceMode =
(e.currentTarget as HTMLSelectElement).value === "slim" ? "slim" : "full";
updateRunnerDraft((draft) => ({ ...draft, evidenceMode }));
});
root.querySelector<HTMLSelectElement>("#runtime-pair")?.addEventListener("change", (e) => {
const runtimePair: RunnerSelection["runtimePair"] =
(e.currentTarget as HTMLSelectElement).value === "openclaw,codex"
? ["openclaw", "codex"]
: null;
updateRunnerDraft((draft) => ({ ...draft, runtimePair }));
});
root.querySelector<HTMLSelectElement>("#runtime-pair-lane")?.addEventListener("change", (e) => {
const value = (e.currentTarget as HTMLSelectElement).value;
const runtimePairLane =
value === "core" || value === "extended" || value === "soak" ? value : null;
updateRunnerDraft((draft) => ({ ...draft, runtimePairLane }));
});
root.querySelector<HTMLSelectElement>("#primary-model")?.addEventListener("change", (e) => {
const primaryModel = (e.currentTarget as HTMLSelectElement).value;
updateRunnerDraft((d) => ({

View File

@@ -51,7 +51,22 @@ export async function postJson<T>(path: string, body: unknown): Promise<T> {
response,
path,
).catch(() => ({}));
throw new Error(payload.error || `${response.status} ${response.statusText}`);
throw new QaLabHttpError(
payload.error || `${response.status} ${response.statusText}`,
response.status,
payload,
);
}
return await readJsonResponse<T>(response, path);
}
export class QaLabHttpError extends Error {
constructor(
message: string,
readonly status: number,
readonly payload: unknown,
) {
super(message);
this.name = "QaLabHttpError";
}
}

View File

@@ -511,6 +511,7 @@ select {
.scenario-item {
display: flex;
flex-shrink: 0;
align-items: center;
gap: 10px;
padding: 10px 12px;

View File

@@ -93,7 +93,17 @@ export function renderSidebar(state: UiState): string {
const realModels = state.bootstrap?.runnerCatalog.real ?? [];
const modelOptions =
selection?.providerMode === "live-frontier" && realModels.length > 0 ? realModels : MOCK_MODELS;
const selectedIds = new Set(selection?.scenarioIds ?? []);
const plan = state.runnerPlanOverride ?? state.bootstrap?.runner.plan ?? null;
const resolvedIds = plan?.selectedScenarios.map((scenario) => scenario.id) ?? [];
const selectedIds = new Set(
selection?.scenarioIds ?? (state.runnerDraftDirty ? [] : resolvedIds),
);
const profiles = state.bootstrap?.runnerCatalog.profiles ?? [];
const channels = state.bootstrap?.runnerCatalog.channels ?? [];
const hasRunnableSelection =
selection?.scenarioIds === null
? Boolean(selection.profile)
: Boolean(selection?.scenarioIds.length);
return `
<aside class="sidebar${state.sidebarCollapsed ? " is-collapsed" : ""}">
@@ -106,6 +116,17 @@ export function renderSidebar(state: UiState): string {
state.sidebarPanel === "config"
? `<div class="sidebar-section sidebar-panel-body">
<div class="sidebar-section-title"><h3>Configuration</h3></div>
<div class="config-field">
<span class="config-label">Profile</span>
<select id="run-profile"${isRunning ? " disabled" : ""}>
${profiles
.map(
(profile) =>
`<option value="${esc(profile.id)}"${selection?.profile === profile.id ? " selected" : ""}>${esc(profile.id)}</option>`,
)
.join("")}
</select>
</div>
<div class="config-field">
<span class="config-label">Provider lane</span>
<select id="provider-mode"${isRunning ? " disabled" : ""}>
@@ -113,6 +134,52 @@ export function renderSidebar(state: UiState): string {
<option value="live-frontier"${selection?.providerMode === "live-frontier" ? " selected" : ""}>Real frontier providers</option>
</select>
</div>
<div class="config-field">
<span class="config-label">Channel driver</span>
<select id="channel-driver"${isRunning ? " disabled" : ""}>
<option value="qa-channel"${selection?.channelDriver === "qa-channel" ? " selected" : ""}>Synthetic QA channel</option>
<option value="crabline"${selection?.channelDriver === "crabline" ? " selected" : ""}>Crabline channel driver</option>
<option value="live"${selection?.channelDriver === "live" ? " selected" : ""}>Real channels</option>
</select>
</div>
<div class="config-field">
<span class="config-label">Execution channel</span>
<select id="execution-channel"${isRunning ? " disabled" : ""}>
<option value=""${selection?.channel ? "" : " selected"}>Catalog/default</option>
${channels
.map(
(channel) =>
`<option value="${esc(channel)}"${selection?.channel === channel ? " selected" : ""}>${esc(channel)}</option>`,
)
.join("")}
</select>
</div>
<div class="config-field">
<span class="config-label">Evidence mode</span>
<select id="evidence-mode"${isRunning ? " disabled" : ""}>
<option value="full"${selection?.evidenceMode === "full" ? " selected" : ""}>Full</option>
<option value="slim"${selection?.evidenceMode === "slim" ? " selected" : ""}>Slim</option>
</select>
</div>
<div class="config-field">
<span class="config-label">Runtime pair</span>
<select id="runtime-pair"${isRunning ? " disabled" : ""}>
<option value=""${selection?.runtimePair ? "" : " selected"}>Single runtime</option>
<option value="openclaw,codex"${selection?.runtimePair ? " selected" : ""}>OpenClaw × Codex</option>
</select>
</div>
<div class="config-field">
<span class="config-label">Runtime-pair lane</span>
<select id="runtime-pair-lane"${isRunning ? " disabled" : ""}>
<option value=""${selection?.runtimePairLane ? "" : " selected"}>Profile/default</option>
${(["core", "extended", "soak"] as const)
.map(
(lane) =>
`<option value="${lane}"${selection?.runtimePairLane === lane ? " selected" : ""}>${lane}</option>`,
)
.join("")}
</select>
</div>
${renderModelSelect({
id: "primary-model",
label: "Primary model",
@@ -143,10 +210,10 @@ export function renderSidebar(state: UiState): string {
? `<div class="sidebar-panel-body">${run || runner ? renderRunStatus(state) : '<div class="sidebar-section"><div class="text-dimmed text-sm">No run data yet.</div></div>'}</div>`
: `<div class="sidebar-section sidebar-scenarios sidebar-panel-body">
<div class="sidebar-section-title">
<h3>Scenarios (${selectedIds.size}/${scenarios.length})</h3>
<h3>Scenarios (${selection?.scenarioIds === null ? "profile" : selectedIds.size}/${scenarios.length})</h3>
<div class="btn-group">
<button class="btn-sm btn-ghost" data-action="select-all-scenarios"${isRunning ? " disabled" : ""}>All</button>
<button class="btn-sm btn-ghost" data-action="clear-scenarios"${isRunning ? " disabled" : ""}>None</button>
<button class="btn-sm btn-ghost" data-action="clear-scenarios"${isRunning ? " disabled" : ""}>Profile</button>
</div>
</div>
<div class="scenario-scroll">
@@ -160,7 +227,7 @@ export function renderSidebar(state: UiState): string {
<span class="${statusDotClass(status)}"></span>
<div class="scenario-item-info">
<span class="scenario-item-title">${esc(s.title)}</span>
<span class="scenario-item-meta">${esc(s.surface)} · ${esc(s.id)}</span>
<span class="scenario-item-meta">${esc(s.surface)} · ${esc(s.execution?.kind ?? "flow")} · ${esc(s.id)}</span>
</div>
</label>`;
})
@@ -171,8 +238,8 @@ export function renderSidebar(state: UiState): string {
<!-- Actions -->
<div class="sidebar-actions">
<button class="btn-primary" data-action="run-suite"${isRunning || !selectedIds.size || state.busy ? " disabled" : ""}>
Run ${selectedIds.size} scenario${selectedIds.size === 1 ? "" : "s"}
<button class="btn-primary" data-action="run-suite"${isRunning || !hasRunnableSelection || state.busy ? " disabled" : ""}>
${selection?.scenarioIds === null ? `Resolve & run ${esc(selection.profile)}` : `Run ${selectedIds.size} scenario${selectedIds.size === 1 ? "" : "s"}`}
</button>
<div class="btn-row">
<button data-action="self-check"${isRunning || state.busy ? " disabled" : ""}>Self-check</button>
@@ -185,6 +252,7 @@ export function renderSidebar(state: UiState): string {
function renderRunStatus(state: UiState): string {
const run = state.scenarioRun;
const runner = state.bootstrap?.runner ?? null;
const plan = state.runnerPlanOverride ?? (state.runnerDraftDirty ? null : (runner?.plan ?? null));
if (!run && !runner) {
return "";
}
@@ -206,6 +274,9 @@ function renderRunStatus(state: UiState): string {
: ""
}
<div class="run-meta">
${plan ? `<strong>Resolved plan:</strong> ${plan.selectedScenarios.length} selected · ${esc(plan.executionKinds.join(", ") || "none")}` : ""}
${plan?.exclusions.length ? `<br>${plan.exclusions.length} excluded: ${esc(plan.exclusions.map((item) => `${item.scenarioId} (${item.reasons.join(", ")})`).join("; "))}` : ""}
${plan?.errors.length ? `<br><span style="color:var(--danger)">${esc(plan.errors.join(" "))}</span>` : ""}
${runner?.startedAt ? `Started ${esc(formatIso(runner.startedAt))}` : ""}
${runner?.finishedAt ? `<br>Finished ${esc(formatIso(runner.finishedAt))}` : ""}
${runner?.error ? `<br><span style="color:var(--danger)">${esc(runner.error)}</span>` : ""}

View File

@@ -68,6 +68,7 @@ function evidenceState(overrides: Partial<UiState> = {}): UiState {
latestReport: null,
runnerDraft: null,
runnerDraftDirty: false,
runnerPlanOverride: null,
scenarioRun: null,
selectedCaptureEventKey: null,
selectedCaptureSessionIds: [],

View File

@@ -1,3 +1,9 @@
import type {
QaLabExecutionKind,
QaLabResolvedRunPlan,
QaLabRunnerSnapshot,
QaLabRunSelection,
} from "../../runner-contract.js";
import type {
QaEvidenceArtifactView,
QaEvidenceGalleryEntryView,
@@ -82,6 +88,11 @@ export type SeedScenario = {
successCriteria: string[];
docsRefs?: string[];
codeRefs?: string[];
execution?: {
kind?: QaLabExecutionKind;
channel?: string;
};
runtimePairLane?: "core" | "extended" | "soak";
};
export type Bootstrap = {
@@ -101,6 +112,13 @@ export type Bootstrap = {
runnerCatalog: {
status: "loading" | "ready" | "failed";
real: RunnerModelOption[];
channels: string[];
profiles: Array<{
id: string;
evidenceMode: "full" | "slim";
channelDriver: "qa-channel" | "crabline" | "live";
categoryIds: string[];
}>;
};
};
@@ -136,28 +154,9 @@ type ScenarioRun = {
};
};
export type RunnerSelection = {
providerMode: "mock-openai" | "live-frontier";
primaryModel: string;
alternateModel: string;
fastMode: boolean;
scenarioIds: string[];
};
type RunnerSnapshot = {
status: "idle" | "running" | "completed" | "failed";
selection: RunnerSelection;
startedAt?: string;
finishedAt?: string;
artifacts: null | {
evidencePath: string;
outputDir: string;
reportPath: string;
summaryPath: string;
watchUrl: string;
};
error: string | null;
};
export type RunnerSelection = QaLabRunSelection;
export type RunnerResolvedPlan = QaLabResolvedRunPlan;
type RunnerSnapshot = QaLabRunnerSnapshot;
export type RunnerModelOption = {
key: string;
@@ -370,6 +369,7 @@ export type UiState = {
activeTab: TabId;
runnerDraft: RunnerSelection | null;
runnerDraftDirty: boolean;
runnerPlanOverride: RunnerResolvedPlan | null;
composer: {
conversationKind: "direct" | "channel";
conversationId: string;