mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 22:17:00 +00:00
fix(qa-lab): unify runner control plane (#112838)
This commit is contained in:
@@ -110,6 +110,23 @@ the QA Lab page where an operator or automation loop can give the agent a QA
|
||||
mission, observe real channel behavior, and record what worked, failed, or
|
||||
stayed blocked.
|
||||
|
||||
The Runner's Scenarios panel can launch flow, Playwright, Vitest, and script
|
||||
catalog entries together. **Profile** uses the taxonomy-owned membership plan;
|
||||
checking scenarios creates an explicit override, while **Profile** in the
|
||||
Scenarios panel returns to server-resolved profile membership.
|
||||
|
||||
Config also exposes **Provider lane**, primary and alternate models,
|
||||
**Execution channel**, **Channel driver**, **Evidence mode**, **Runtime pair**,
|
||||
and **Runtime-pair lane** (`core`, `extended`, or `soak`). Provider/model,
|
||||
runtime, and channel-driver choices remain independent: for example, Real
|
||||
frontier providers can use the Crabline channel driver, and Synthetic (mock) can
|
||||
use Real channels. The server resolves taxonomy membership, provider/model
|
||||
eligibility, declared `execution.channel`, runtime-pair-lane membership, and
|
||||
supported execution kinds before launch. The Run panel shows the selected
|
||||
execution kinds plus explicit exclusions or errors. Unknown, empty explicit,
|
||||
profile-incompatible, or lane-incompatible selections fail closed instead of
|
||||
being replaced by a default suite.
|
||||
|
||||
For faster QA Lab UI iteration without rebuilding the Docker image each time,
|
||||
start the stack with a bind-mounted QA Lab bundle:
|
||||
|
||||
|
||||
60
extensions/qa-lab/runner-contract.ts
Normal file
60
extensions/qa-lab/runner-contract.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import type { QaProviderMode } from "./src/providers/index.js";
|
||||
import type { QaRuntimePairLane, QaScenarioExecution } from "./src/scenario-catalog.js";
|
||||
import type {
|
||||
QaScorecardChannelDriver,
|
||||
QaScorecardEvidenceMode,
|
||||
} from "./src/scorecard-taxonomy.js";
|
||||
|
||||
export type QaLabExecutionKind = NonNullable<QaScenarioExecution["kind"]>;
|
||||
|
||||
export type QaLabRunSelection = {
|
||||
profile: string;
|
||||
channel: string | null;
|
||||
channelDriver: QaScorecardChannelDriver;
|
||||
evidenceMode: QaScorecardEvidenceMode;
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode: boolean;
|
||||
runtimePair: ["openclaw", "codex"] | null;
|
||||
runtimePairLane: QaRuntimePairLane | null;
|
||||
scenarioIds: string[] | null;
|
||||
};
|
||||
|
||||
export type QaLabResolvedRunPlan = {
|
||||
status: "ready" | "invalid";
|
||||
profile: string;
|
||||
explicitScenarioSelection: boolean;
|
||||
selectedScenarios: Array<{
|
||||
id: string;
|
||||
title: string;
|
||||
executionKind: QaLabExecutionKind;
|
||||
declaredChannel: string | null;
|
||||
effectiveChannel: string | null;
|
||||
}>;
|
||||
executionKinds: QaLabExecutionKind[];
|
||||
exclusions: Array<{
|
||||
scenarioId: string;
|
||||
executionKind: QaLabExecutionKind;
|
||||
reasons: string[];
|
||||
}>;
|
||||
errors: string[];
|
||||
};
|
||||
|
||||
type QaLabRunArtifacts = {
|
||||
outputDir: string;
|
||||
evidencePath: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
};
|
||||
|
||||
export type QaLabRunnerSnapshot = {
|
||||
status: "idle" | "running" | "completed" | "failed";
|
||||
selection: QaLabRunSelection;
|
||||
plan: QaLabResolvedRunPlan | null;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
artifacts: QaLabRunArtifacts | null;
|
||||
error: string | null;
|
||||
};
|
||||
@@ -556,7 +556,7 @@ describe("qa cli runtime", () => {
|
||||
expectWriteContains(stdoutWrite, "QA run profile: all; categories: 1; scenarios:");
|
||||
});
|
||||
|
||||
it("filters QA-channel-pinned scenarios from the Crabline smoke profile", async () => {
|
||||
it("filters QA-channel-pinned scenarios from an implicit Crabline smoke profile", async () => {
|
||||
runQaSuite.mockImplementationOnce(async () => {
|
||||
await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
|
||||
return flowSuiteRuntimeResult({
|
||||
@@ -568,12 +568,11 @@ describe("qa cli runtime", () => {
|
||||
await runQaProfileCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
profile: "smoke-ci",
|
||||
scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"],
|
||||
});
|
||||
|
||||
const suiteArgs = mockFirstObjectArg(runQaSuite);
|
||||
expect(suiteArgs.channelDriver).toBe("crabline");
|
||||
expect(suiteArgs.scenarioIds).toEqual(["channel-top-level-reply-shape"]);
|
||||
expect(suiteArgs.scenarioIds).toContain("channel-top-level-reply-shape");
|
||||
expect(suiteArgs.scenarioIds).not.toEqual(
|
||||
expect.arrayContaining([
|
||||
"instruction-followthrough-repo-contract",
|
||||
@@ -603,6 +602,20 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects explicit profile selections with an incompatible scenario", async () => {
|
||||
await expect(
|
||||
runQaProfileCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
profile: "smoke-ci",
|
||||
scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"],
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"qa run --qa-profile smoke-ci cannot run explicitly selected scenario(s): control-ui-qa-channel-image-roundtrip (channelDriver=qa-channel).",
|
||||
);
|
||||
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dispatches the Matrix restart scenario through the Crabline smoke profile", async () => {
|
||||
await runQaProfileCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -773,15 +786,20 @@ describe("qa cli runtime", () => {
|
||||
expect(runQaMultipass).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects runtime-pair execution for live adapters", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
it("keeps runtime-pair execution independent from live adapters", async () => {
|
||||
await runQaSuiteCommand({
|
||||
channelDriver: "live",
|
||||
channel: "telegram",
|
||||
runtimePair: "openclaw,codex",
|
||||
});
|
||||
|
||||
expect(runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
channelDriver: "live",
|
||||
channel: "telegram",
|
||||
runtimePair: "openclaw,codex",
|
||||
channelId: "telegram",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
}),
|
||||
).rejects.toThrow("--runtime-pair is not supported with a live QA adapter.");
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
);
|
||||
});
|
||||
|
||||
it("loads contributed adapters without preselecting a scenario channel", async () => {
|
||||
|
||||
@@ -46,6 +46,10 @@ import { startQaLabServer } from "./lab-server.js";
|
||||
import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js";
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
import { runQaMultipass } from "./multipass.runtime.js";
|
||||
import {
|
||||
resolveQaRunProfileExecutionSelection,
|
||||
resolveQaRunProfileMembership,
|
||||
} from "./profile-planning.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
import {
|
||||
QA_FRONTIER_PARITY_BASELINE_LABEL,
|
||||
@@ -68,6 +72,10 @@ import {
|
||||
type QaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "./run-config.js";
|
||||
import {
|
||||
resolveQaRuntimePairLaneScenarioIds,
|
||||
resolveQaRuntimePairScenarioSupport,
|
||||
} from "./runtime-pair-lane-selection.js";
|
||||
import type { RuntimeId } from "./runtime-parity.js";
|
||||
import {
|
||||
QA_RUNTIME_PAIR_LANES,
|
||||
@@ -80,7 +88,6 @@ import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js"
|
||||
import {
|
||||
qaScorecardChannelDriverSchema,
|
||||
readQaScorecardTaxonomyReport,
|
||||
type QaScorecardCategoryCoverageReport,
|
||||
type QaScorecardChannelDriver,
|
||||
type QaScorecardEvidenceMode,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
@@ -311,76 +318,6 @@ function parseQaRuntimePairLaneFilters(input: string[] | undefined): QaRuntimePa
|
||||
return rawValues as QaRuntimePairLane[];
|
||||
}
|
||||
|
||||
function resolveQaRuntimePairLaneScenarioIds(params: {
|
||||
channelDriver?: QaScorecardChannelDriver | null;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
scenarioIds: string[];
|
||||
runtimePairLanes: readonly QaRuntimePairLane[];
|
||||
runtimePair: boolean;
|
||||
}): {
|
||||
scenarioIds: string[];
|
||||
excludedLaneScenarios: string[];
|
||||
excludedNonFlowScenarios: string[];
|
||||
} {
|
||||
if (params.runtimePairLanes.length === 0) {
|
||||
return {
|
||||
scenarioIds: params.scenarioIds,
|
||||
excludedLaneScenarios: [],
|
||||
excludedNonFlowScenarios: [],
|
||||
};
|
||||
}
|
||||
const laneSet = new Set(params.runtimePairLanes);
|
||||
const matchingScenarios = readQaScenarioPack().scenarios.filter(
|
||||
(scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane),
|
||||
);
|
||||
if (matchingScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
const compatibleScenarios = params.runtimePair
|
||||
? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow")
|
||||
: matchingScenarios;
|
||||
const laneCompatibleScenarios = compatibleScenarios.filter((scenario) =>
|
||||
scenarioMatchesQaProviderLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
channelDriver: params.channelDriver,
|
||||
channel: scenario.execution.channel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
);
|
||||
const excludedLaneScenarios = compatibleScenarios
|
||||
.filter((scenario) => !laneCompatibleScenarios.includes(scenario))
|
||||
.map((scenario) => scenario.id);
|
||||
const excludedNonFlowScenarios = params.runtimePair
|
||||
? matchingScenarios
|
||||
.filter((scenario) => scenario.execution.kind !== "flow")
|
||||
.map((scenario) => `${scenario.id} (${scenario.execution.kind})`)
|
||||
: [];
|
||||
if (compatibleScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
scenarioIds: uniqueStrings([
|
||||
...params.scenarioIds,
|
||||
...laneCompatibleScenarios.map((scenario) => scenario.id),
|
||||
]),
|
||||
excludedLaneScenarios,
|
||||
excludedNonFlowScenarios,
|
||||
};
|
||||
}
|
||||
|
||||
function rejectNonFlowScenarioIds(params: {
|
||||
option: "--runner multipass" | "--runtime-pair";
|
||||
scenarioIds: readonly string[];
|
||||
@@ -392,12 +329,13 @@ function rejectNonFlowScenarioIds(params: {
|
||||
const scenarioById = new Map(
|
||||
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
|
||||
);
|
||||
const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
|
||||
const selectedScenarios = scenarioIds.flatMap((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
return scenario && scenario.execution.kind !== "flow"
|
||||
? [`${scenario.id} (${scenario.execution.kind})`]
|
||||
: [];
|
||||
return scenario ? [scenario] : [];
|
||||
});
|
||||
const nonFlowScenarios = resolveQaRuntimePairScenarioSupport(
|
||||
selectedScenarios,
|
||||
).excludedScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`);
|
||||
if (nonFlowScenarios.length > 0) {
|
||||
throw new Error(
|
||||
`${params.option} requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
|
||||
@@ -742,43 +680,30 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
|
||||
if (!profileReport) {
|
||||
throw new Error(`taxonomy.yaml does not define QA run profile ${profile}.`);
|
||||
}
|
||||
const categories = scorecardReport.categories.filter((category) =>
|
||||
qaScorecardCategoryMatchesRunProfile(category, {
|
||||
const membership = resolveQaRunProfileMembership(
|
||||
{
|
||||
profile,
|
||||
surface: opts.surface,
|
||||
category: opts.category,
|
||||
}),
|
||||
scenarioIds: opts.scenarioIds,
|
||||
},
|
||||
{ scenarios: scenarioPack.scenarios, scorecardReport },
|
||||
);
|
||||
const categories = membership.categories;
|
||||
if (categories.length === 0) {
|
||||
throw new Error(formatQaRunProfileNoMatchMessage(opts));
|
||||
}
|
||||
|
||||
const scenarioBySourcePath = new Map(
|
||||
scenarioPack.scenarios.map((scenario) => [scenario.sourcePath, scenario] as const),
|
||||
);
|
||||
const taxonomyScenariosForCategories = uniqueStrings(
|
||||
categories.flatMap((category) => category.scenarioRefs),
|
||||
)
|
||||
.map((scenarioRef) => scenarioBySourcePath.get(scenarioRef))
|
||||
.filter((scenario): scenario is NonNullable<typeof scenario> => scenario !== undefined);
|
||||
const requestedScenarioIds = uniqueStrings(
|
||||
(opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean),
|
||||
);
|
||||
const taxonomyScenarios =
|
||||
requestedScenarioIds.length === 0
|
||||
? taxonomyScenariosForCategories
|
||||
: taxonomyScenariosForCategories.filter((scenario) =>
|
||||
requestedScenarioIds.includes(scenario.id),
|
||||
);
|
||||
const taxonomyScenarios = membership.selectedScenarios;
|
||||
if (requestedScenarioIds.length > 0 && taxonomyScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${requestedScenarioIds.join(",")}.`,
|
||||
);
|
||||
}
|
||||
const matchedScenarioIds = new Set(taxonomyScenarios.map((scenario) => scenario.id));
|
||||
const missingScenarioIds = requestedScenarioIds.filter(
|
||||
(scenarioId) => !matchedScenarioIds.has(scenarioId),
|
||||
);
|
||||
const missingScenarioIds = membership.excludedScenarioIds;
|
||||
if (missingScenarioIds.length > 0) {
|
||||
throw new Error(
|
||||
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${missingScenarioIds.join(",")}.`,
|
||||
@@ -787,29 +712,23 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
|
||||
const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile);
|
||||
const normalizedProviderMode = normalizeQaProviderMode(providerMode);
|
||||
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode);
|
||||
const scenarios = taxonomyScenarios.filter((scenario) => {
|
||||
// qa-channel is the built-in harness channel, so another driver cannot implement it.
|
||||
if (
|
||||
scenario.execution.channel === "qa-channel" &&
|
||||
profileReport.channelDriver !== "qa-channel"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const channel =
|
||||
profileReport.channelDriver === "qa-channel"
|
||||
? "qa-channel"
|
||||
: (scenario.execution.channel ??
|
||||
(profileReport.channelDriver === "crabline"
|
||||
? OPENCLAW_CRABLINE_DEFAULT_CHANNEL
|
||||
: undefined));
|
||||
return scenarioMatchesQaProviderLane({
|
||||
scenario,
|
||||
providerMode: normalizedProviderMode,
|
||||
primaryModel,
|
||||
channelDriver: profileReport.channelDriver,
|
||||
channel,
|
||||
});
|
||||
const executionSelection = resolveQaRunProfileExecutionSelection({
|
||||
scenarios: taxonomyScenarios,
|
||||
providerMode: normalizedProviderMode,
|
||||
primaryModel,
|
||||
channelDriver: profileReport.channelDriver,
|
||||
defaultChannel:
|
||||
profileReport.channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined,
|
||||
});
|
||||
if (requestedScenarioIds.length > 0 && executionSelection.excludedScenarios.length > 0) {
|
||||
const exclusions = executionSelection.excludedScenarios
|
||||
.map(({ scenario, reasons }) => `${scenario.id} (${reasons.join(", ")})`)
|
||||
.join(", ");
|
||||
throw new Error(
|
||||
`qa run --qa-profile ${profile} cannot run explicitly selected scenario(s): ${exclusions}.`,
|
||||
);
|
||||
}
|
||||
const scenarios = executionSelection.selectedScenarios;
|
||||
if (scenarios.length === 0) {
|
||||
throw new Error(
|
||||
`qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`,
|
||||
@@ -898,25 +817,6 @@ function defaultQaRunProfileProviderMode(profile: string): QaProviderModeInput {
|
||||
return profile === "smoke-ci" ? "mock-openai" : DEFAULT_QA_LIVE_PROVIDER_MODE;
|
||||
}
|
||||
|
||||
function qaScorecardCategoryMatchesRunProfile(
|
||||
category: QaScorecardCategoryCoverageReport,
|
||||
opts: { profile: string; surface?: string; category?: string },
|
||||
): boolean {
|
||||
if (!category.profiles.includes(opts.profile)) {
|
||||
return false;
|
||||
}
|
||||
if (opts.surface?.trim()) {
|
||||
const surface = opts.surface.trim();
|
||||
if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (opts.category?.trim() && category.id !== opts.category.trim()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function formatQaRunProfileNoMatchMessage(
|
||||
opts: Pick<QaProfileCommandOptions, "profile" | "surface" | "category">,
|
||||
) {
|
||||
@@ -967,8 +867,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
|
||||
});
|
||||
const runtimePairLanes = parseQaRuntimePairLaneFilters(opts.runtimePairLane);
|
||||
const runtimePairLaneSelection = resolveQaRuntimePairLaneScenarioIds({
|
||||
channel: opts.channel,
|
||||
channelDriver,
|
||||
claudeCliAuthMode,
|
||||
defaultChannel: channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined,
|
||||
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
|
||||
providerMode,
|
||||
scenarioIds: explicitScenarioIds,
|
||||
@@ -981,12 +883,12 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
|
||||
}
|
||||
if (runtimePairLaneSelection.excludedNonFlowScenarios.length > 0) {
|
||||
process.stderr.write(
|
||||
`QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.join(", ")}\n`,
|
||||
`QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}\n`,
|
||||
);
|
||||
}
|
||||
if (runtimePairLaneSelection.excludedLaneScenarios.length > 0) {
|
||||
process.stderr.write(
|
||||
`QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.join(", ")}\n`,
|
||||
`QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}\n`,
|
||||
);
|
||||
}
|
||||
const allowFailures = opts.allowFailures === true;
|
||||
@@ -1064,9 +966,6 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
|
||||
if (runner === "multipass" && liveChannelId) {
|
||||
throw new Error("--channel-driver live with --channel requires --runner host.");
|
||||
}
|
||||
if (runtimePair && liveChannelId) {
|
||||
throw new Error("--runtime-pair is not supported with a live QA adapter.");
|
||||
}
|
||||
if (runner === "multipass") {
|
||||
rejectNonFlowScenarioIds({ option: "--runner multipass", scenarioIds });
|
||||
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
||||
|
||||
@@ -75,6 +75,9 @@ describe("qa docker harness", () => {
|
||||
OPENCLAW_STATE_DIR: "/tmp/openclaw/state",
|
||||
});
|
||||
expect(services["qa-lab"]?.volumes).toContain("./state:/opt/openclaw-scaffold:ro");
|
||||
expect(services["qa-lab"]?.volumes).toContain(
|
||||
`${path.relative(outputDir, "/repo/openclaw/taxonomy.yaml").split(path.sep).join("/")}:/app/taxonomy.yaml:ro`,
|
||||
);
|
||||
expect(compose).toContain(' - "127.0.0.1:18889:18789"');
|
||||
expect(compose).toContain(' - "127.0.0.1:43124:43123"');
|
||||
expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro");
|
||||
@@ -154,7 +157,7 @@ describe("qa docker harness", () => {
|
||||
|
||||
expect(result.imageName).toBe("openclaw:qa-local-prebaked");
|
||||
expect(calls).toEqual([
|
||||
"docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
|
||||
"docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=acpx qa-channel qa-lab -f Dockerfile . @/repo/openclaw",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -177,10 +180,14 @@ describe("qa docker harness", () => {
|
||||
|
||||
const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8");
|
||||
const services = parseComposeServices(compose);
|
||||
expect(compose).toContain('OPENCLAW_EXTENSIONS: "acpx qa-channel qa-lab"');
|
||||
expect(services["qa-mock-openai"]?.build?.context).toBe("../repo #hash");
|
||||
expect(services["qa-lab"]?.volumes).toContain(
|
||||
"../repo #hash/extensions/qa-lab/web/dist:/opt/openclaw-qa-lab-ui:ro",
|
||||
);
|
||||
expect(services["qa-lab"]?.volumes).toContain(
|
||||
"../repo #hash/taxonomy.yaml:/app/taxonomy.yaml:ro",
|
||||
);
|
||||
expect(services["openclaw-qa-gateway"]?.volumes).toContain(
|
||||
"../repo #hash:/opt/openclaw-repo:ro",
|
||||
);
|
||||
|
||||
@@ -12,6 +12,9 @@ import { buildQaGatewayConfig } from "./qa-gateway-config.js";
|
||||
|
||||
const QA_LAB_INTERNAL_PORT = 43123;
|
||||
const QA_LAB_UI_OVERLAY_DIR = "/opt/openclaw-qa-lab-ui";
|
||||
// The QA config enables ACPX. Bake the external plugin so ephemeral Gateways do
|
||||
// not block startup on a network install before their health deadline.
|
||||
const QA_DOCKER_PLUGIN_SELECTION = "acpx qa-channel qa-lab";
|
||||
|
||||
function toPosixRelative(fromDir: string, toPath: string): string {
|
||||
return path.relative(fromDir, toPath).split(path.sep).join("/");
|
||||
@@ -31,7 +34,7 @@ function renderImageBlock(params: {
|
||||
return ` image: ${params.imageName}\n`;
|
||||
}
|
||||
const context = toPosixRelative(params.outputDir, params.repoRoot) || ".";
|
||||
return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "qa-channel qa-lab"\n`;
|
||||
return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "${QA_DOCKER_PLUGIN_SELECTION}"\n`;
|
||||
}
|
||||
|
||||
function renderCompose(params: {
|
||||
@@ -46,6 +49,10 @@ function renderCompose(params: {
|
||||
}) {
|
||||
const imageBlock = renderImageBlock(params);
|
||||
const repoMount = toPosixRelative(params.outputDir, params.repoRoot) || ".";
|
||||
const taxonomyMount = toPosixRelative(
|
||||
params.outputDir,
|
||||
path.join(params.repoRoot, "taxonomy.yaml"),
|
||||
);
|
||||
const qaLabUiMount = toPosixRelative(
|
||||
params.outputDir,
|
||||
path.join(params.repoRoot, "extensions", "qa-lab", "web", "dist"),
|
||||
@@ -84,6 +91,7 @@ ${imageBlock} pull_policy: never
|
||||
- "127.0.0.1:${params.qaLabPort}:${QA_LAB_INTERNAL_PORT}"
|
||||
volumes:
|
||||
- ./state:/opt/openclaw-scaffold:ro
|
||||
- ${yamlDoubleQuoted(`${taxonomyMount}:/app/taxonomy.yaml:ro`)}
|
||||
${params.bindUiDist ? ` - ${yamlDoubleQuoted(`${qaLabUiMount}:${QA_LAB_UI_OVERLAY_DIR}:ro`)}\n` : ""} healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
@@ -190,7 +198,7 @@ Files:
|
||||
Suggested flow:
|
||||
|
||||
1. Build the prebaked image once:
|
||||
- \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="qa-channel qa-lab" -f Dockerfile .\`
|
||||
- \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="${QA_DOCKER_PLUGIN_SELECTION}" -f Dockerfile .\`
|
||||
2. Start the stack:
|
||||
- \`docker compose -f docker-compose.qa.yml up${params.usePrebuiltImage ? "" : " --build"} -d\`
|
||||
3. Open the QA dashboard:
|
||||
@@ -354,7 +362,7 @@ export async function buildQaDockerHarnessImage(
|
||||
"-t",
|
||||
imageName,
|
||||
"--build-arg",
|
||||
"OPENCLAW_EXTENSIONS=qa-channel qa-lab",
|
||||
`OPENCLAW_EXTENSIONS=${QA_DOCKER_PLUGIN_SELECTION}`,
|
||||
"-f",
|
||||
"Dockerfile",
|
||||
".",
|
||||
|
||||
@@ -14,6 +14,22 @@ const qaChannelMock = vi.hoisted(() => ({
|
||||
startAccount: vi.fn(),
|
||||
}));
|
||||
|
||||
const suiteLaunchMock = vi.hoisted(() => ({
|
||||
runQaSuite: vi.fn(),
|
||||
}));
|
||||
const liveTransportMock = vi.hoisted(() => ({
|
||||
adapterFactories: [{ id: "live-test-factory", matches: vi.fn(), create: vi.fn() }],
|
||||
listAdapterFactories: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./suite-launch.runtime.js", () => ({
|
||||
runQaSuite: suiteLaunchMock.runQaSuite,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/cli.js", () => ({
|
||||
listLiveTransportQaAdapterFactories: liveTransportMock.listAdapterFactories,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/qa-channel", () => ({
|
||||
qaChannelPlugin: {
|
||||
config: {
|
||||
@@ -161,6 +177,10 @@ async function startQaLabServerForTest(params?: QaLabServerStartParams) {
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
suiteLaunchMock.runQaSuite.mockReset();
|
||||
liveTransportMock.listAdapterFactories.mockReset();
|
||||
liveTransportMock.listAdapterFactories.mockReturnValue(liveTransportMock.adapterFactories);
|
||||
liveTransportMock.adapterFactories[0]!.matches.mockReset();
|
||||
qaChannelMock.resolveAccount.mockReset();
|
||||
qaChannelMock.resolveAccount.mockImplementation((_cfg: unknown, accountId: string) => ({
|
||||
accountId,
|
||||
@@ -186,10 +206,10 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
captureMock.reset();
|
||||
while (cleanups.length > 0) {
|
||||
await cleanups.pop()?.();
|
||||
}
|
||||
captureMock.reset();
|
||||
});
|
||||
|
||||
function isRetryableLocalFetchError(error: unknown) {
|
||||
@@ -324,6 +344,333 @@ async function createQaLabRepoRootFixture(params?: {
|
||||
}
|
||||
|
||||
describe("qa-lab server", () => {
|
||||
it("dispatches explicit mixed-kind selections through the suite planner", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
suiteLaunchMock.runQaSuite.mockResolvedValue({
|
||||
executionKind: "suite",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-evidence.json",
|
||||
outputDir: "/tmp/qa-output",
|
||||
report: "# QA report\n",
|
||||
reportPath: "/tmp/qa-report.md",
|
||||
scenarios: [],
|
||||
summaryPath: "/tmp/qa-summary.json",
|
||||
},
|
||||
});
|
||||
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
const launch = (await response.json()) as {
|
||||
plan: {
|
||||
executionKinds: string[];
|
||||
selectedScenarios: Array<{
|
||||
id: string;
|
||||
declaredChannel: string | null;
|
||||
effectiveChannel: string | null;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
expect(launch.plan.executionKinds).toEqual(["flow", "playwright"]);
|
||||
expect(launch.plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([
|
||||
"dm-chat-baseline",
|
||||
"browser-talk-start-stop",
|
||||
]);
|
||||
expect(launch.plan.selectedScenarios).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: "dm-chat-baseline",
|
||||
declaredChannel: null,
|
||||
effectiveChannel: "telegram",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: "browser-talk-start-stop",
|
||||
declaredChannel: null,
|
||||
effectiveChannel: null,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
|
||||
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
channelDriver: "crabline",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
|
||||
}),
|
||||
);
|
||||
expect(liveTransportMock.listAdapterFactories).not.toHaveBeenCalled();
|
||||
await vi.waitFor(async () => {
|
||||
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
|
||||
runner: {
|
||||
status: string;
|
||||
selection: { scenarioIds: string[] };
|
||||
artifacts: { watchUrl: string };
|
||||
};
|
||||
};
|
||||
expect(bootstrap.runner.status).toBe("completed");
|
||||
expect(bootstrap.runner.artifacts.watchUrl).toBe(lab.baseUrl);
|
||||
expect(bootstrap.runner.selection.scenarioIds).toEqual([
|
||||
"dm-chat-baseline",
|
||||
"browser-talk-start-stop",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps mock providers independent from real channel adapters", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
suiteLaunchMock.runQaSuite.mockResolvedValue({
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-evidence.json",
|
||||
outputDir: "/tmp/qa-output",
|
||||
report: "# QA report\n",
|
||||
reportPath: "/tmp/qa-report.md",
|
||||
scenarios: [],
|
||||
summaryPath: "/tmp/qa-summary.json",
|
||||
watchUrl: "http://runtime-watch.invalid",
|
||||
},
|
||||
});
|
||||
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
channelDriver: "live",
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
|
||||
expect(liveTransportMock.listAdapterFactories).toHaveBeenCalledTimes(1);
|
||||
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
adapterFactories: liveTransportMock.adapterFactories,
|
||||
channelDriver: "live",
|
||||
providerMode: "mock-openai",
|
||||
}),
|
||||
);
|
||||
await vi.waitFor(async () => {
|
||||
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
|
||||
runner: { status: string; artifacts: { watchUrl: string } };
|
||||
};
|
||||
expect(bootstrap.runner.status).toBe("completed");
|
||||
expect(bootstrap.runner.artifacts.watchUrl).toBe("http://runtime-watch.invalid");
|
||||
});
|
||||
});
|
||||
|
||||
it("allows only one concurrent request to commit a resolved suite plan", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
let finishSuite: ((value: unknown) => void) | undefined;
|
||||
suiteLaunchMock.runQaSuite.mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
finishSuite = resolve;
|
||||
}),
|
||||
);
|
||||
const request = () =>
|
||||
fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
}),
|
||||
});
|
||||
|
||||
const responses = await Promise.all([request(), request()]);
|
||||
|
||||
expect(
|
||||
responses.map((response) => response.status).toSorted((left, right) => left - right),
|
||||
).toEqual([202, 409]);
|
||||
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1);
|
||||
finishSuite?.({
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-evidence.json",
|
||||
outputDir: "/tmp/qa-output",
|
||||
report: "# QA report\n",
|
||||
reportPath: "/tmp/qa-report.md",
|
||||
scenarios: [],
|
||||
summaryPath: "/tmp/qa-summary.json",
|
||||
},
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as {
|
||||
runner: { status: string };
|
||||
};
|
||||
expect(bootstrap.runner.status).toBe("completed");
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects empty and unknown explicit selections before dispatch", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
for (const scenarioIds of [[], ["missing-scenario"]]) {
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ scenarioIds }),
|
||||
});
|
||||
expect(response.status).toBe(400);
|
||||
}
|
||||
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns the resolved runtime-pair-lane plan and launches it with independent live transport", async () => {
|
||||
liveTransportMock.adapterFactories[0]!.matches.mockReturnValue(true);
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
suiteLaunchMock.runQaSuite.mockResolvedValue({
|
||||
executionKind: "flow",
|
||||
result: {
|
||||
evidencePath: "/tmp/qa-evidence.json",
|
||||
outputDir: "/tmp/qa-output",
|
||||
report: "# QA report\n",
|
||||
reportPath: "/tmp/qa-report.md",
|
||||
scenarios: [],
|
||||
summaryPath: "/tmp/qa-summary.json",
|
||||
watchUrl: lab.baseUrl,
|
||||
},
|
||||
});
|
||||
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
profile: "all",
|
||||
channel: "telegram",
|
||||
channelDriver: "live",
|
||||
evidenceMode: "slim",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "core",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(202);
|
||||
const payload = (await response.json()) as {
|
||||
plan: {
|
||||
executionKinds: string[];
|
||||
exclusions: Array<{ scenarioId: string }>;
|
||||
selectedScenarios: Array<{ id: string }>;
|
||||
};
|
||||
};
|
||||
expect(payload.plan.executionKinds).toEqual(["flow"]);
|
||||
expect(payload.plan.selectedScenarios.map((scenario) => scenario.id)).toContain(
|
||||
"runtime-first-hour-20-turn",
|
||||
);
|
||||
expect(payload.plan.exclusions.map((exclusion) => exclusion.scenarioId)).toContain(
|
||||
"codex-plugin-cold-install",
|
||||
);
|
||||
await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1));
|
||||
expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
adapterFactories: liveTransportMock.adapterFactories,
|
||||
channelDriver: "live",
|
||||
channelId: "telegram",
|
||||
evidenceMode: "slim",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns explicit exclusions and errors without launching unsupported execution kinds", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
scenarioIds: ["browser-talk-start-stop"],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
const payload = (await response.json()) as {
|
||||
error: string;
|
||||
plan: {
|
||||
status: string;
|
||||
exclusions: Array<{ scenarioId: string; reasons: string[] }>;
|
||||
errors: string[];
|
||||
};
|
||||
};
|
||||
expect(payload.plan.status).toBe("invalid");
|
||||
expect(payload.plan.exclusions).toEqual([
|
||||
expect.objectContaining({
|
||||
scenarioId: "browser-talk-start-stop",
|
||||
reasons: ["runtimePair requires execution.kind=flow"],
|
||||
}),
|
||||
]);
|
||||
expect(payload.error).toContain("Explicit QA scenario selection is not runnable");
|
||||
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("enforces explicit execution.channel through the shared suite channel planner", async () => {
|
||||
const lab = await startQaLabServerForTest();
|
||||
cleanups.push(async () => {
|
||||
await lab.stop();
|
||||
});
|
||||
|
||||
const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
profile: "all",
|
||||
channel: "telegram",
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["matrix-room-block-streaming"],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
const payload = (await response.json()) as {
|
||||
plan: { exclusions: Array<{ scenarioId: string; reasons: string[] }> };
|
||||
};
|
||||
expect(payload.plan.exclusions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ scenarioId: "matrix-room-block-streaming" }),
|
||||
]),
|
||||
);
|
||||
expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleans up capture state when embedded gateway setup fails", async () => {
|
||||
qaChannelMock.resolveAccount.mockImplementationOnce(() => {
|
||||
throw new Error("embedded setup failed");
|
||||
@@ -396,7 +743,8 @@ describe("qa-lab server", () => {
|
||||
kickoffTask: string;
|
||||
scenarios: Array<{ id: string; title: string; execution?: { kind?: string } }>;
|
||||
defaults: { conversationId: string; senderId: string };
|
||||
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } };
|
||||
runner: { status: string; selection: { providerMode: string; scenarioIds: string[] | null } };
|
||||
runnerCatalog: { channels: string[]; profiles: Array<{ id: string }> };
|
||||
};
|
||||
expect(bootstrap.defaults.conversationId).toBe("qa-operator");
|
||||
expect(bootstrap.defaults.senderId).toBe("qa-operator");
|
||||
@@ -406,13 +754,14 @@ describe("qa-lab server", () => {
|
||||
expect(bootstrap.scenarios.length).toBeGreaterThanOrEqual(10);
|
||||
expect(bootstrap.scenarios.map((scenario) => scenario.id)).toContain("dm-chat-baseline");
|
||||
expect(bootstrap.runner.status).toBe("idle");
|
||||
expect(bootstrap.runner.selection.providerMode).toBe("live-frontier");
|
||||
const flowScenarioIds = bootstrap.scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
expect(bootstrap.runner.selection.scenarioIds).toEqual(flowScenarioIds);
|
||||
expect(bootstrap.runner.selection.providerMode).toBe("mock-openai");
|
||||
expect(bootstrap.runner.selection.scenarioIds).toBeNull();
|
||||
expect(bootstrap.runnerCatalog.profiles.map((profile) => profile.id)).toEqual([
|
||||
"smoke-ci",
|
||||
"release",
|
||||
"all",
|
||||
]);
|
||||
expect(bootstrap.runnerCatalog.channels).toContain("qa-channel");
|
||||
|
||||
const startupStatus = (await (
|
||||
await fetchWithRetry(`${lab.baseUrl}/api/capture/startup-status`)
|
||||
|
||||
@@ -50,12 +50,15 @@ import type {
|
||||
} from "./lab-server.types.js";
|
||||
import type { QaRunnerModelOption } from "./model-catalog.runtime.js";
|
||||
import { createQaChannelGatewayConfig } from "./qa-channel-transport.js";
|
||||
import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
|
||||
import {
|
||||
createIdleQaRunnerSnapshot,
|
||||
createQaRunOutputDir,
|
||||
normalizeQaRunSelection,
|
||||
resolveQaLabRunPlan,
|
||||
} from "./run-config.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js";
|
||||
import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js";
|
||||
|
||||
type QaLabBootstrapDefaults = {
|
||||
@@ -300,10 +303,53 @@ export async function startQaLabServer(
|
||||
let latestReport: QaLabLatestReport | null = null;
|
||||
let latestScenarioRun: QaLabScenarioRun | null = null;
|
||||
const scenarioCatalog = readQaBootstrapScenarioCatalog();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(scenarioCatalog.scenarios);
|
||||
const runnerChannels = [
|
||||
...new Set(
|
||||
scenarioCatalog.scenarios
|
||||
.map((scenario) => scenario.execution.channel)
|
||||
.filter((channel): channel is string => Boolean(channel)),
|
||||
),
|
||||
].toSorted();
|
||||
const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget);
|
||||
let runnerModelOptions: QaRunnerModelOption[] = [];
|
||||
let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading";
|
||||
let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios);
|
||||
const resolveServerRunPlan = async (
|
||||
selection: ReturnType<typeof normalizeQaRunSelection>,
|
||||
adapterFactories?: readonly QaTransportAdapterFactory[],
|
||||
) => {
|
||||
const crabline =
|
||||
selection.channelDriver === "crabline" ? await import("@openclaw/crabline") : undefined;
|
||||
return resolveQaLabRunPlan({
|
||||
selection,
|
||||
scenarios: scenarioCatalog.scenarios,
|
||||
scorecardReport,
|
||||
defaultChannel:
|
||||
crabline?.OPENCLAW_CRABLINE_DEFAULT_CHANNEL ??
|
||||
(selection.channelDriver === "qa-channel" ? "qa-channel" : undefined),
|
||||
...(crabline
|
||||
? {
|
||||
supportsChannel: (channel: string) => {
|
||||
try {
|
||||
crabline.resolveOpenClawCrablineChannelDriverSelection({ channel });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
}
|
||||
: adapterFactories
|
||||
? {
|
||||
supportsChannel: (channel: string) =>
|
||||
adapterFactories.some((factory) =>
|
||||
factory.matches({ channelId: channel, driver: "live" }),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
};
|
||||
let runnerSnapshot = createIdleQaRunnerSnapshot(scorecardReport.profiles);
|
||||
runnerSnapshot.plan = await resolveServerRunPlan(runnerSnapshot.selection);
|
||||
let activeSuiteRun: Promise<void> | null = null;
|
||||
let controlUiProxyTarget = params?.controlUiProxyTarget?.trim()
|
||||
? new URL(params.controlUiProxyTarget)
|
||||
@@ -431,6 +477,8 @@ export async function startQaLabServer(
|
||||
runnerCatalog: {
|
||||
status: runnerModelCatalogStatus,
|
||||
real: runnerModelOptions,
|
||||
profiles: scorecardReport.profiles,
|
||||
channels: runnerChannels,
|
||||
},
|
||||
});
|
||||
return;
|
||||
@@ -681,17 +729,52 @@ export async function startQaLabServer(
|
||||
writeError(res, 409, "QA suite run already in progress");
|
||||
return;
|
||||
}
|
||||
const selection = normalizeQaRunSelection(
|
||||
await readQaJsonBody(req),
|
||||
scenarioCatalog.scenarios,
|
||||
);
|
||||
let selection: ReturnType<typeof normalizeQaRunSelection>;
|
||||
let plan: ReturnType<typeof resolveQaLabRunPlan>;
|
||||
let adapterFactories: readonly QaTransportAdapterFactory[] | undefined;
|
||||
try {
|
||||
selection = normalizeQaRunSelection(
|
||||
await readQaJsonBody(req),
|
||||
scenarioCatalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
adapterFactories =
|
||||
selection.channelDriver === "live"
|
||||
? (await import("./live-transports/cli.js")).listLiveTransportQaAdapterFactories()
|
||||
: undefined;
|
||||
plan = await resolveServerRunPlan(selection, adapterFactories);
|
||||
} catch (error) {
|
||||
writeError(res, 400, error);
|
||||
return;
|
||||
}
|
||||
if (plan.status === "invalid") {
|
||||
writeJson(res, 400, {
|
||||
error: plan.errors.join(" "),
|
||||
plan,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (activeSuiteRun) {
|
||||
writeError(res, 409, "QA suite run already in progress");
|
||||
return;
|
||||
}
|
||||
state.reset();
|
||||
latestReport = null;
|
||||
latestScenarioRun = null;
|
||||
const startedAt = new Date().toISOString();
|
||||
latestScenarioRun = withQaLabRunCounts({
|
||||
kind: "suite",
|
||||
status: "running",
|
||||
startedAt,
|
||||
scenarios: plan.selectedScenarios.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
name: scenario.title,
|
||||
status: "pending",
|
||||
})),
|
||||
});
|
||||
runnerSnapshot = {
|
||||
status: "running",
|
||||
selection,
|
||||
plan,
|
||||
startedAt,
|
||||
finishedAt: undefined,
|
||||
artifacts: null,
|
||||
@@ -699,38 +782,86 @@ export async function startQaLabServer(
|
||||
};
|
||||
activeSuiteRun = (async () => {
|
||||
try {
|
||||
const { runQaFlowSuite } = await import("./suite.js");
|
||||
const result = await runQaFlowSuite({
|
||||
const [{ runQaSuite }, channelDriverSelection] = await Promise.all([
|
||||
import("./suite-launch.runtime.js"),
|
||||
selection.channelDriver === "crabline" && selection.channel
|
||||
? import("@openclaw/crabline").then((module) =>
|
||||
module.resolveOpenClawCrablineChannelDriverSelection({
|
||||
channel: selection.channel!,
|
||||
}),
|
||||
)
|
||||
: Promise.resolve(undefined),
|
||||
]);
|
||||
const runtimeResult = await runQaSuite({
|
||||
lab: labHandle ?? undefined,
|
||||
startLab: startQaLabServer,
|
||||
repoRoot,
|
||||
outputDir: createQaRunOutputDir(repoRoot),
|
||||
channelDriver: selection.channelDriver,
|
||||
...(adapterFactories ? { adapterFactories } : {}),
|
||||
...(selection.channelDriver === "live" && selection.channel
|
||||
? { channelId: selection.channel }
|
||||
: {}),
|
||||
...(channelDriverSelection ? { channelDriverSelection } : {}),
|
||||
evidenceMode: selection.evidenceMode,
|
||||
providerMode: selection.providerMode,
|
||||
primaryModel: selection.primaryModel,
|
||||
alternateModel: selection.alternateModel,
|
||||
scenarioIds: selection.scenarioIds,
|
||||
fastMode: selection.fastMode,
|
||||
scenarioIds: plan.selectedScenarios.map((scenario) => scenario.id),
|
||||
...(selection.runtimePair ? { runtimePair: selection.runtimePair } : {}),
|
||||
});
|
||||
const result = runtimeResult.result;
|
||||
const finishedAt = new Date().toISOString();
|
||||
latestReport = {
|
||||
outputPath: result.reportPath,
|
||||
markdown: result.report,
|
||||
generatedAt: finishedAt,
|
||||
};
|
||||
runnerSnapshot = {
|
||||
status: "completed",
|
||||
selection,
|
||||
plan,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
finishedAt,
|
||||
artifacts: {
|
||||
outputDir: result.outputDir,
|
||||
evidencePath: result.evidencePath,
|
||||
reportPath: result.reportPath,
|
||||
summaryPath: result.summaryPath,
|
||||
watchUrl: result.watchUrl,
|
||||
watchUrl:
|
||||
"watchUrl" in result && typeof result.watchUrl === "string"
|
||||
? result.watchUrl
|
||||
: (labHandle?.baseUrl ?? publicBaseUrl),
|
||||
},
|
||||
error: null,
|
||||
};
|
||||
} catch (error) {
|
||||
const finishedAt = new Date().toISOString();
|
||||
const message = formatErrorMessage(error);
|
||||
latestScenarioRun = withQaLabRunCounts({
|
||||
kind: "suite",
|
||||
status: "completed",
|
||||
startedAt,
|
||||
finishedAt,
|
||||
scenarios: (latestScenarioRun?.scenarios ?? []).map((scenario) =>
|
||||
scenario.status === "pending" || scenario.status === "running"
|
||||
? Object.assign({}, scenario, {
|
||||
status: "fail" as const,
|
||||
details: message,
|
||||
finishedAt,
|
||||
})
|
||||
: scenario,
|
||||
),
|
||||
});
|
||||
runnerSnapshot = {
|
||||
status: "failed",
|
||||
selection,
|
||||
plan,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
finishedAt,
|
||||
artifacts: null,
|
||||
error: formatErrorMessage(error),
|
||||
error: message,
|
||||
};
|
||||
} finally {
|
||||
activeSuiteRun = null;
|
||||
@@ -738,6 +869,7 @@ export async function startQaLabServer(
|
||||
})();
|
||||
writeJson(res, 202, {
|
||||
ok: true,
|
||||
plan,
|
||||
runner: runnerSnapshot,
|
||||
});
|
||||
return;
|
||||
|
||||
146
extensions/qa-lab/src/profile-planning.ts
Normal file
146
extensions/qa-lab/src/profile-planning.ts
Normal file
@@ -0,0 +1,146 @@
|
||||
// Qa Lab plugin module owns canonical taxonomy profile membership planning.
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import { describeQaProviderLaneMismatches } from "./scenario-lane.js";
|
||||
import {
|
||||
readQaScorecardTaxonomyReport,
|
||||
type QaScorecardCategoryCoverageReport,
|
||||
type QaScorecardTaxonomyReport,
|
||||
type QaScorecardChannelDriver,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
|
||||
type QaRunProfileMembership = {
|
||||
categories: QaScorecardCategoryCoverageReport[];
|
||||
excludedScenarioIds: string[];
|
||||
profile: QaScorecardTaxonomyReport["profiles"][number];
|
||||
profileScenarios: QaSeedScenarioWithSource[];
|
||||
selectedScenarios: QaSeedScenarioWithSource[];
|
||||
};
|
||||
|
||||
type QaRunProfileExecutionSelection = {
|
||||
excludedScenarios: Array<{ scenario: QaSeedScenarioWithSource; reasons: string[] }>;
|
||||
selectedScenarios: QaSeedScenarioWithSource[];
|
||||
};
|
||||
|
||||
function categoryMatchesRunProfile(
|
||||
category: QaScorecardCategoryCoverageReport,
|
||||
opts: { profile: string; surface?: string; category?: string },
|
||||
): boolean {
|
||||
if (!category.profiles.includes(opts.profile)) {
|
||||
return false;
|
||||
}
|
||||
if (opts.surface?.trim()) {
|
||||
const surface = opts.surface.trim();
|
||||
if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return !opts.category?.trim() || category.id === opts.category.trim();
|
||||
}
|
||||
|
||||
export function resolveQaRunProfileMembership(
|
||||
opts: {
|
||||
profile: string;
|
||||
surface?: string;
|
||||
category?: string;
|
||||
scenarioIds?: readonly string[];
|
||||
},
|
||||
source?: {
|
||||
scenarios?: QaSeedScenarioWithSource[];
|
||||
scorecardReport?: QaScorecardTaxonomyReport;
|
||||
},
|
||||
): QaRunProfileMembership {
|
||||
const scenarios = source?.scenarios ?? readQaScenarioPack().scenarios;
|
||||
const scorecardReport = source?.scorecardReport ?? readQaScorecardTaxonomyReport(scenarios);
|
||||
const profileId = opts.profile.trim();
|
||||
const profile = scorecardReport.profiles.find((entry) => entry.id === profileId);
|
||||
if (!profile) {
|
||||
const profileIds = scorecardReport.profiles.map((entry) => entry.id);
|
||||
if (profileIds.length === 0) {
|
||||
throw new Error("taxonomy.yaml does not define QA run profiles.");
|
||||
}
|
||||
throw new Error(
|
||||
`QA run profile must be one of ${profileIds.join(", ")}, got "${opts.profile}".`,
|
||||
);
|
||||
}
|
||||
const categories = scorecardReport.categories.filter((category) =>
|
||||
categoryMatchesRunProfile(category, {
|
||||
profile: profileId,
|
||||
surface: opts.surface,
|
||||
category: opts.category,
|
||||
}),
|
||||
);
|
||||
const scenarioBySourcePath = new Map(
|
||||
scenarios.map((scenario) => [scenario.sourcePath, scenario] as const),
|
||||
);
|
||||
const profileScenarios = uniqueStrings(categories.flatMap((category) => category.scenarioRefs))
|
||||
.map((scenarioRef) => scenarioBySourcePath.get(scenarioRef))
|
||||
.filter((scenario): scenario is QaSeedScenarioWithSource => scenario !== undefined);
|
||||
const requestedScenarioIds = uniqueStrings(
|
||||
(opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean),
|
||||
);
|
||||
if (requestedScenarioIds.length === 0) {
|
||||
return {
|
||||
categories,
|
||||
excludedScenarioIds: [],
|
||||
profile,
|
||||
profileScenarios,
|
||||
selectedScenarios: profileScenarios,
|
||||
};
|
||||
}
|
||||
const requestedScenarioIdSet = new Set(requestedScenarioIds);
|
||||
const selectedScenarios = profileScenarios.filter((scenario) =>
|
||||
requestedScenarioIdSet.has(scenario.id),
|
||||
);
|
||||
const selectedScenarioIdSet = new Set(selectedScenarios.map((scenario) => scenario.id));
|
||||
return {
|
||||
categories,
|
||||
excludedScenarioIds: requestedScenarioIds.filter(
|
||||
(scenarioId) => !selectedScenarioIdSet.has(scenarioId),
|
||||
),
|
||||
profile,
|
||||
profileScenarios,
|
||||
selectedScenarios,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveQaRunProfileExecutionSelection(params: {
|
||||
scenarios: readonly QaSeedScenarioWithSource[];
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
channelDriver: QaScorecardChannelDriver;
|
||||
channel?: string | null;
|
||||
defaultChannel?: string;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}): QaRunProfileExecutionSelection {
|
||||
const selectedScenarios: QaSeedScenarioWithSource[] = [];
|
||||
const excludedScenarios: QaRunProfileExecutionSelection["excludedScenarios"] = [];
|
||||
for (const scenario of params.scenarios) {
|
||||
const reasons: string[] = [];
|
||||
// qa-channel is the built-in harness channel, so another driver cannot implement it.
|
||||
if (scenario.execution.channel === "qa-channel" && params.channelDriver !== "qa-channel") {
|
||||
reasons.push("channelDriver=qa-channel");
|
||||
}
|
||||
reasons.push(
|
||||
...describeQaProviderLaneMismatches({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
channelDriver: params.channelDriver,
|
||||
channel:
|
||||
params.channelDriver === "qa-channel"
|
||||
? "qa-channel"
|
||||
: (params.channel ?? scenario.execution.channel ?? params.defaultChannel),
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
);
|
||||
if (reasons.length > 0) {
|
||||
excludedScenarios.push({ scenario, reasons: uniqueStrings(reasons) });
|
||||
} else {
|
||||
selectedScenarios.push(scenario);
|
||||
}
|
||||
}
|
||||
return { excludedScenarios, selectedScenarios };
|
||||
}
|
||||
@@ -11,14 +11,29 @@ vi.mock("./model-selection.runtime.js", () => ({
|
||||
defaultQaRuntimeModelForMode,
|
||||
}));
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import {
|
||||
resolveQaRunProfileExecutionSelection,
|
||||
resolveQaRunProfileMembership,
|
||||
} from "./profile-planning.js";
|
||||
import {
|
||||
createIdleQaRunnerSnapshot,
|
||||
createQaRunOutputDir,
|
||||
normalizeQaRunSelection,
|
||||
resolveQaLabRunPlan,
|
||||
type QaProviderModeInput,
|
||||
} from "./run-config.js";
|
||||
import { readQaScenarioPack } from "./scenario-catalog.js";
|
||||
import {
|
||||
readQaScorecardTaxonomyReport,
|
||||
type QaScorecardTaxonomyReport,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
|
||||
const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier");
|
||||
const profiles: QaScorecardTaxonomyReport["profiles"] = [
|
||||
{ id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] },
|
||||
{ id: "release", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
|
||||
{ id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
|
||||
];
|
||||
|
||||
const scenarios = [
|
||||
{
|
||||
@@ -58,17 +73,25 @@ describe("qa run config", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("creates a live-by-default selection that arms flow scenarios", () => {
|
||||
expect(normalizeQaRunSelection({}, scenarios)).toEqual({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
fastMode: true,
|
||||
scenarioIds: ["dm-chat-baseline", "thread-lifecycle"],
|
||||
});
|
||||
it("creates a canonical smoke-profile request without copying profile membership", () => {
|
||||
const expected = {
|
||||
profile: "smoke-ci",
|
||||
channel: null,
|
||||
channelDriver: "crabline",
|
||||
evidenceMode: "slim",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
fastMode: false,
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: null,
|
||||
} as const;
|
||||
expect(normalizeQaRunSelection({}, scenarios, profiles)).toEqual(expected);
|
||||
expect(normalizeQaRunSelection({ profile: null }, scenarios, profiles)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("normalizes live selections and filters unknown scenario ids", () => {
|
||||
it("normalizes live selections and deduplicates scenario ids", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
@@ -76,15 +99,45 @@ describe("qa run config", () => {
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "",
|
||||
fastMode: false,
|
||||
scenarioIds: ["thread-lifecycle", "missing", "thread-lifecycle"],
|
||||
scenarioIds: ["thread-lifecycle", "thread-lifecycle"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toEqual({
|
||||
profile: "all",
|
||||
channel: null,
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: DEFAULT_LIVE_FRONTIER_MODEL,
|
||||
fastMode: true,
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["thread-lifecycle"],
|
||||
});
|
||||
});
|
||||
|
||||
it("applies the canonical profile execution defaults to explicit scenario requests", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{ profile: "all", scenarioIds: ["thread-lifecycle"] },
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toMatchObject({
|
||||
profile: "all",
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
scenarioIds: ["thread-lifecycle"],
|
||||
});
|
||||
expect(
|
||||
normalizeQaRunSelection({ scenarioIds: ["thread-lifecycle"] }, scenarios, profiles),
|
||||
).toMatchObject({
|
||||
profile: "all",
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
scenarioIds: ["thread-lifecycle"],
|
||||
});
|
||||
});
|
||||
@@ -96,46 +149,464 @@ describe("qa run config", () => {
|
||||
providerMode: "live-openai",
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toThrow("unknown QA provider mode: live-openai");
|
||||
});
|
||||
|
||||
it("falls back to all scenarios when selection would otherwise be empty", () => {
|
||||
const snapshot = createIdleQaRunnerSnapshot(scenarios);
|
||||
it("keeps implicit profile membership server-owned and rejects an explicit empty selection", () => {
|
||||
const snapshot = createIdleQaRunnerSnapshot(profiles);
|
||||
expect(snapshot.status).toBe("idle");
|
||||
expect(snapshot.selection.scenarioIds).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
scenarioIds: [],
|
||||
},
|
||||
scenarios,
|
||||
).scenarioIds,
|
||||
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
|
||||
expect(snapshot.selection.scenarioIds).toBeNull();
|
||||
expect(snapshot.plan).toBeNull();
|
||||
expect(() => normalizeQaRunSelection({ scenarioIds: [] }, scenarios, profiles)).toThrow(
|
||||
"scenarioIds must be a non-empty array",
|
||||
);
|
||||
});
|
||||
|
||||
it("filters non-flow scenarios from lab runner selections", () => {
|
||||
it("preserves explicit non-flow scenarios for the mixed-kind suite planner", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
scenarioIds: ["control-ui-chat-flow-playwright", "thread-lifecycle"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
).scenarioIds,
|
||||
).toEqual(["thread-lifecycle"]);
|
||||
).toEqual(["control-ui-chat-flow-playwright", "thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("fails closed on unknown explicit scenario ids", () => {
|
||||
expect(() =>
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
scenarioIds: ["thread-lifecycle", "missing"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toThrow("unknown QA scenario id(s): missing");
|
||||
});
|
||||
|
||||
it("fails closed on malformed explicit profiles", () => {
|
||||
for (const profile of [false, ""] as const) {
|
||||
expect(() =>
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
profile,
|
||||
scenarioIds: ["thread-lifecycle"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toThrow("QA runner profile must be a non-empty string");
|
||||
}
|
||||
});
|
||||
|
||||
it("normalizes the channel driver independently from the provider lane", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toMatchObject({ channelDriver: "crabline", providerMode: "live-frontier" });
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
channelDriver: "live",
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toMatchObject({ channelDriver: "live", providerMode: "mock-openai" });
|
||||
});
|
||||
|
||||
it("normalizes every server-owned control-plane axis", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channel: " Telegram ",
|
||||
channelDriver: "crabline",
|
||||
evidenceMode: "slim",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "extended",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toMatchObject({
|
||||
profile: "all",
|
||||
channel: "telegram",
|
||||
channelDriver: "crabline",
|
||||
evidenceMode: "slim",
|
||||
providerMode: "live-frontier",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "extended",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a reversed runtime pair instead of silently changing its order", () => {
|
||||
expect(() =>
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
runtimePair: ["codex", "openclaw"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toThrow('runtimePair must be ["openclaw", "codex"]');
|
||||
});
|
||||
|
||||
it("shares implicit profile membership and eligibility with the canonical profile planner", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const selection = normalizeQaRunSelection({}, catalog.scenarios, scorecardReport.profiles);
|
||||
const membership = resolveQaRunProfileMembership(
|
||||
{ profile: selection.profile },
|
||||
{ scenarios: catalog.scenarios, scorecardReport },
|
||||
);
|
||||
const expected = resolveQaRunProfileExecutionSelection({
|
||||
scenarios: membership.selectedScenarios,
|
||||
providerMode: selection.providerMode,
|
||||
primaryModel: selection.primaryModel,
|
||||
channelDriver: selection.channelDriver,
|
||||
channel: selection.channel,
|
||||
});
|
||||
|
||||
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
|
||||
|
||||
expect(plan.status).toBe("ready");
|
||||
expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual(
|
||||
expected.selectedScenarios.map((scenario) => scenario.id),
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves mixed execution kinds and reports runtime-pair-lane exclusions", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const mixedSelection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
const mixedPlan = resolveQaLabRunPlan({
|
||||
selection: mixedSelection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
});
|
||||
expect(mixedPlan.status).toBe("ready");
|
||||
expect(mixedPlan.executionKinds).toEqual(["flow", "playwright"]);
|
||||
|
||||
const pairSelection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "core",
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
const pairPlan = resolveQaLabRunPlan({
|
||||
selection: pairSelection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
});
|
||||
expect(pairPlan.status).toBe("ready");
|
||||
expect(pairPlan.selectedScenarios.map((scenario) => scenario.id)).toContain(
|
||||
"runtime-first-hour-20-turn",
|
||||
);
|
||||
expect(pairPlan.exclusions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
scenarioId: "codex-plugin-cold-install",
|
||||
reasons: ["runtimePair requires execution.kind=flow"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("validates explicit runtime-pair-lane selections without expanding them", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const selectedId = "runtime-first-hour-20-turn";
|
||||
const selection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "core",
|
||||
scenarioIds: [selectedId],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
|
||||
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
|
||||
|
||||
expect(plan.status).toBe("ready");
|
||||
expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([selectedId]);
|
||||
|
||||
const outsideLaneSelection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "mock-openai",
|
||||
runtimePairLane: "core",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
const outsideLanePlan = resolveQaLabRunPlan({
|
||||
selection: outsideLaneSelection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
});
|
||||
expect(outsideLanePlan.status).toBe("invalid");
|
||||
expect(outsideLanePlan.selectedScenarios).toEqual([]);
|
||||
expect(outsideLanePlan.exclusions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
scenarioId: "dm-chat-baseline",
|
||||
reasons: ["runtimePairLane=core"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not apply live-adapter eligibility to non-flow executions", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const selection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "live",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["browser-talk-start-stop"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
|
||||
const plan = resolveQaLabRunPlan({
|
||||
selection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
supportsChannel: () => false,
|
||||
});
|
||||
|
||||
expect(plan.status).toBe("ready");
|
||||
expect(plan.selectedScenarios).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "browser-talk-start-stop",
|
||||
effectiveChannel: null,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves canonical unresolved live dispatch and rejects unsupported driver channels", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const liveSelection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "live",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
const unresolvedLivePlan = resolveQaLabRunPlan({
|
||||
selection: liveSelection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
supportsChannel: () => true,
|
||||
});
|
||||
expect(unresolvedLivePlan.status).toBe("ready");
|
||||
expect(unresolvedLivePlan.selectedScenarios).toEqual([
|
||||
expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: null }),
|
||||
]);
|
||||
|
||||
const crablineSelection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channel: "qa-channel",
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
const unsupportedCrablinePlan = resolveQaLabRunPlan({
|
||||
selection: crablineSelection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
defaultChannel: "telegram",
|
||||
supportsChannel: (channel) => channel === "telegram",
|
||||
});
|
||||
expect(unsupportedCrablinePlan.status).toBe("invalid");
|
||||
expect(unsupportedCrablinePlan.exclusions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
scenarioId: "dm-chat-baseline",
|
||||
reasons: ["unsupported crabline channel=qa-channel"],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed when an explicit scenario conflicts with execution.channel", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const scenario = catalog.scenarios.find(
|
||||
(entry) => entry.execution.channel && entry.execution.channel !== "qa-channel",
|
||||
);
|
||||
expect(scenario).toBeDefined();
|
||||
const selection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: [scenario!.id],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
|
||||
const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport });
|
||||
|
||||
expect(plan.status).toBe("invalid");
|
||||
expect(plan.exclusions).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ scenarioId: scenario!.id })]),
|
||||
);
|
||||
expect(plan.errors.join(" ")).toContain("Explicit QA scenario selection is not runnable");
|
||||
});
|
||||
|
||||
it("returns an invalid resolved plan for a qa-channel execution override", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const selection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channel: "telegram",
|
||||
channelDriver: "qa-channel",
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
|
||||
const plan = resolveQaLabRunPlan({
|
||||
selection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
defaultChannel: "qa-channel",
|
||||
});
|
||||
|
||||
expect(plan.status).toBe("invalid");
|
||||
expect(plan.errors).toContain(
|
||||
"An execution channel requires channelDriver=crabline or channelDriver=live.",
|
||||
);
|
||||
expect(plan.selectedScenarios).toEqual([
|
||||
expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: "qa-channel" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("distinguishes declared constraints from the suite-resolved effective channel", () => {
|
||||
const catalog = readQaScenarioPack();
|
||||
const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios);
|
||||
const selection = normalizeQaRunSelection(
|
||||
{
|
||||
profile: "all",
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
|
||||
},
|
||||
catalog.scenarios,
|
||||
scorecardReport.profiles,
|
||||
);
|
||||
|
||||
const plan = resolveQaLabRunPlan({
|
||||
selection,
|
||||
scenarios: catalog.scenarios,
|
||||
scorecardReport,
|
||||
defaultChannel: "telegram",
|
||||
});
|
||||
|
||||
expect(plan.selectedScenarios).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: "dm-chat-baseline",
|
||||
declaredChannel: null,
|
||||
effectiveChannel: "telegram",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: "browser-talk-start-stop",
|
||||
declaredChannel: null,
|
||||
effectiveChannel: null,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects malformed requests and unknown channel drivers", () => {
|
||||
expect(() => normalizeQaRunSelection(null, scenarios, profiles)).toThrow(
|
||||
"request must be a JSON object",
|
||||
);
|
||||
expect(() =>
|
||||
normalizeQaRunSelection({ channelDriver: "renamed-cli-policy" }, scenarios, profiles),
|
||||
).toThrow("unknown QA channel driver: renamed-cli-policy");
|
||||
});
|
||||
|
||||
it("keeps idle snapshots on static defaults so startup does not inspect auth profiles", () => {
|
||||
defaultQaRuntimeModelForMode.mockReturnValue("openai/gpt-5.6-luna");
|
||||
defaultQaRuntimeModelForMode.mockClear();
|
||||
|
||||
const selection = createIdleQaRunnerSnapshot(scenarios).selection;
|
||||
expect(selection.providerMode).toBe("live-frontier");
|
||||
expect(selection.primaryModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL);
|
||||
expect(selection.alternateModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL);
|
||||
const selection = createIdleQaRunnerSnapshot(profiles).selection;
|
||||
expect(selection.providerMode).toBe("mock-openai");
|
||||
expect(selection.primaryModel).toBe("mock-openai/gpt-5.6-luna");
|
||||
expect(selection.alternateModel).toBe("mock-openai/gpt-5.6-luna-alt");
|
||||
expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed when required canonical profiles are missing", () => {
|
||||
const profilesWithoutSmoke = profiles.filter((profile) => profile.id !== "smoke-ci");
|
||||
const profilesWithoutAll = profiles.filter((profile) => profile.id !== "all");
|
||||
|
||||
expect(() => createIdleQaRunnerSnapshot(profilesWithoutSmoke)).toThrow(
|
||||
"profile table does not define required profile: smoke-ci",
|
||||
);
|
||||
expect(() => normalizeQaRunSelection({}, scenarios, profilesWithoutSmoke)).toThrow(
|
||||
"profile table does not define required profile: smoke-ci",
|
||||
);
|
||||
expect(() =>
|
||||
normalizeQaRunSelection({ scenarioIds: ["dm-chat-baseline"] }, scenarios, profilesWithoutAll),
|
||||
).toThrow("unknown QA run profile: all");
|
||||
});
|
||||
|
||||
it("normalizes aimock selections", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
@@ -146,12 +617,19 @@ describe("qa run config", () => {
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
scenarios,
|
||||
profiles,
|
||||
),
|
||||
).toEqual({
|
||||
profile: "all",
|
||||
channel: null,
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
providerMode: "aimock",
|
||||
primaryModel: "aimock/gpt-5.6-luna",
|
||||
alternateModel: "aimock/gpt-5.6-luna-alt",
|
||||
fastMode: false,
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
});
|
||||
@@ -185,12 +663,18 @@ describe("qa run config", () => {
|
||||
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
|
||||
expect(normalizeQaRunSelection({}, scenarios)).toEqual({
|
||||
expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({
|
||||
profile: "release",
|
||||
channel: null,
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
fastMode: true,
|
||||
scenarioIds: ["dm-chat-baseline", "thread-lifecycle"],
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,8 +2,18 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type {
|
||||
QaLabExecutionKind,
|
||||
QaLabResolvedRunPlan,
|
||||
QaLabRunnerSnapshot,
|
||||
QaLabRunSelection,
|
||||
} from "../runner-contract.js";
|
||||
import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js";
|
||||
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
import {
|
||||
resolveQaRunProfileExecutionSelection,
|
||||
resolveQaRunProfileMembership,
|
||||
} from "./profile-planning.js";
|
||||
import {
|
||||
DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
getQaProvider,
|
||||
@@ -11,35 +21,26 @@ import {
|
||||
normalizeQaProviderMode as normalizeQaProviderModeInput,
|
||||
type QaProviderMode,
|
||||
} from "./providers/index.js";
|
||||
import type { QaSeedScenario } from "./scenario-catalog.js";
|
||||
import {
|
||||
resolveQaRuntimePairLaneScenarioIds,
|
||||
resolveQaRuntimePairScenarioSupport,
|
||||
} from "./runtime-pair-lane-selection.js";
|
||||
import {
|
||||
qaRuntimePairLaneSchema,
|
||||
type QaSeedScenario,
|
||||
type QaSeedScenarioWithSource,
|
||||
} from "./scenario-catalog.js";
|
||||
import {
|
||||
qaScorecardChannelDriverSchema,
|
||||
qaScorecardEvidenceModeSchema,
|
||||
type QaScorecardTaxonomyReport,
|
||||
} from "./scorecard-taxonomy.js";
|
||||
import { resolveQaSuiteScenarioChannels } from "./suite-planning.js";
|
||||
|
||||
export type { QaProviderMode } from "./model-selection.js";
|
||||
export type { QaProviderModeInput } from "./providers/index.js";
|
||||
|
||||
type QaLabRunSelection = {
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode: boolean;
|
||||
scenarioIds: string[];
|
||||
};
|
||||
|
||||
type QaLabRunArtifacts = {
|
||||
outputDir: string;
|
||||
evidencePath: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
};
|
||||
|
||||
type QaLabRunnerSnapshot = {
|
||||
status: "idle" | "running" | "completed" | "failed";
|
||||
selection: QaLabRunSelection;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
artifacts: QaLabRunArtifacts | null;
|
||||
error: string | null;
|
||||
};
|
||||
type QaLabRunProfileOption = QaScorecardTaxonomyReport["profiles"][number];
|
||||
|
||||
export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
return defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined);
|
||||
@@ -51,26 +52,33 @@ function defaultStaticModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
return defaultStaticQaModelForMode(mode, alternate ? { alternate: true } : undefined);
|
||||
}
|
||||
|
||||
function qaLabFlowScenarioIds(scenarios: QaSeedScenario[]) {
|
||||
return scenarios
|
||||
.filter(
|
||||
(scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow",
|
||||
)
|
||||
.map((scenario) => scenario.id);
|
||||
function requireQaRunProfile(profiles: readonly QaLabRunProfileOption[], profileId: string) {
|
||||
const profile = profiles.find((entry) => entry.id === profileId);
|
||||
if (!profile) {
|
||||
throw new Error(`QA runner profile table does not define required profile: ${profileId}`);
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
||||
function createDefaultQaRunSelection(
|
||||
scenarios: QaSeedScenario[],
|
||||
profiles: readonly QaLabRunProfileOption[],
|
||||
options?: { resolveDefaultModel?: QaDefaultModelResolver },
|
||||
): QaLabRunSelection {
|
||||
const providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE;
|
||||
const profile = requireQaRunProfile(profiles, "smoke-ci");
|
||||
const providerMode: QaProviderMode = "mock-openai";
|
||||
const resolveDefaultModel = options?.resolveDefaultModel ?? defaultQaModelForMode;
|
||||
return {
|
||||
profile: profile.id,
|
||||
channel: null,
|
||||
channelDriver: profile.channelDriver,
|
||||
evidenceMode: profile.evidenceMode,
|
||||
providerMode,
|
||||
primaryModel: resolveDefaultModel(providerMode),
|
||||
alternateModel: resolveDefaultModel(providerMode, true),
|
||||
fastMode: true,
|
||||
scenarioIds: qaLabFlowScenarioIds(scenarios),
|
||||
fastMode: getQaProvider(providerMode).kind === "live",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -90,25 +98,143 @@ function normalizeModel(input: unknown, fallback: string) {
|
||||
return value || fallback;
|
||||
}
|
||||
|
||||
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]) {
|
||||
const defaultScenarioIds = qaLabFlowScenarioIds(scenarios);
|
||||
const availableIds = new Set(defaultScenarioIds);
|
||||
const requestedIds = Array.isArray(input)
|
||||
? input
|
||||
.map((value) => (typeof value === "string" ? value.trim() : ""))
|
||||
.filter((value) => value.length > 0)
|
||||
: [];
|
||||
const selectedIds = uniqueStrings(requestedIds.filter((id) => availableIds.has(id)));
|
||||
return selectedIds.length > 0 ? selectedIds : defaultScenarioIds;
|
||||
function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null {
|
||||
if (input === undefined || input === null) {
|
||||
return null;
|
||||
}
|
||||
if (!Array.isArray(input) || input.length === 0) {
|
||||
throw new Error("QA runner scenarioIds must be a non-empty array");
|
||||
}
|
||||
const requestedIds = input.map((value) => {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error("QA runner scenarioIds must contain non-empty strings");
|
||||
}
|
||||
return value.trim();
|
||||
});
|
||||
const selectedIds = uniqueStrings(requestedIds);
|
||||
const availableIds = new Set(scenarios.map((scenario) => scenario.id));
|
||||
const unknownIds = selectedIds.filter((id) => !availableIds.has(id));
|
||||
if (unknownIds.length > 0) {
|
||||
throw new Error(`unknown QA scenario id(s): ${unknownIds.join(", ")}`);
|
||||
}
|
||||
return selectedIds;
|
||||
}
|
||||
|
||||
function normalizeQaChannelDriver(
|
||||
input: unknown,
|
||||
fallback: QaLabRunSelection["channelDriver"],
|
||||
): QaLabRunSelection["channelDriver"] {
|
||||
if (input === undefined || input === null || input === "") {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = qaScorecardChannelDriverSchema.safeParse(input);
|
||||
if (!parsed.success) {
|
||||
const details = typeof input === "string" ? `: ${input}` : "";
|
||||
throw new Error(`unknown QA channel driver${details}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function normalizeQaProfile(
|
||||
input: unknown,
|
||||
profiles: readonly QaLabRunProfileOption[],
|
||||
fallbackProfile?: string,
|
||||
) {
|
||||
const fallback = fallbackProfile ?? requireQaRunProfile(profiles, "smoke-ci").id;
|
||||
// Match the other optional runner controls: null means no override, while any
|
||||
// concrete profile value must be a non-empty string or the request fails closed.
|
||||
if (input !== undefined && input !== null && (typeof input !== "string" || !input.trim())) {
|
||||
throw new Error("QA runner profile must be a non-empty string");
|
||||
}
|
||||
const profile = typeof input === "string" ? input.trim() : fallback;
|
||||
if (!profiles.some((entry) => entry.id === profile)) {
|
||||
throw new Error(
|
||||
`unknown QA run profile: ${profile}; expected one of ${profiles.map((entry) => entry.id).join(", ")}`,
|
||||
);
|
||||
}
|
||||
return profile;
|
||||
}
|
||||
|
||||
function normalizeQaChannel(input: unknown): string | null {
|
||||
if (input === undefined || input === null || input === "") {
|
||||
return null;
|
||||
}
|
||||
if (typeof input !== "string" || !input.trim()) {
|
||||
throw new Error("QA runner channel must be a non-empty string");
|
||||
}
|
||||
return input.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeQaEvidenceMode(
|
||||
input: unknown,
|
||||
fallback: QaLabRunSelection["evidenceMode"],
|
||||
): QaLabRunSelection["evidenceMode"] {
|
||||
if (input === undefined || input === null || input === "") {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = qaScorecardEvidenceModeSchema.safeParse(input);
|
||||
if (!parsed.success) {
|
||||
const details = typeof input === "string" ? `: ${input}` : "";
|
||||
throw new Error(`unknown QA evidence mode${details}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function normalizeQaRuntimePair(input: unknown): QaLabRunSelection["runtimePair"] {
|
||||
if (input === undefined || input === null) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
!Array.isArray(input) ||
|
||||
input.length !== 2 ||
|
||||
!input.every((runtime) => runtime === "openclaw" || runtime === "codex")
|
||||
) {
|
||||
throw new Error('QA runner runtimePair must be ["openclaw", "codex"]');
|
||||
}
|
||||
if (input[0] === input[1]) {
|
||||
throw new Error("QA runner runtimePair must compare two different runtimes");
|
||||
}
|
||||
if (input[0] !== "openclaw" || input[1] !== "codex") {
|
||||
throw new Error('QA runner runtimePair must be ["openclaw", "codex"]');
|
||||
}
|
||||
return ["openclaw", "codex"];
|
||||
}
|
||||
|
||||
function normalizeQaRuntimePairLane(input: unknown): QaLabRunSelection["runtimePairLane"] {
|
||||
if (input === undefined || input === null || input === "") {
|
||||
return null;
|
||||
}
|
||||
const parsed = qaRuntimePairLaneSchema.safeParse(input);
|
||||
if (!parsed.success) {
|
||||
const details = typeof input === "string" ? `: ${input}` : "";
|
||||
throw new Error(`unknown QA runtime-pair lane${details}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
export function normalizeQaRunSelection(
|
||||
input: unknown,
|
||||
scenarios: QaSeedScenario[],
|
||||
profiles: readonly QaLabRunProfileOption[],
|
||||
): QaLabRunSelection {
|
||||
const payload = input && typeof input === "object" ? (input as Record<string, unknown>) : {};
|
||||
const providerMode = normalizeQaProviderMode(payload.providerMode);
|
||||
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
||||
throw new Error("QA runner request must be a JSON object");
|
||||
}
|
||||
const payload = input as Record<string, unknown>;
|
||||
const profile = normalizeQaProfile(
|
||||
payload.profile,
|
||||
profiles,
|
||||
Array.isArray(payload.scenarioIds) ? "all" : undefined,
|
||||
);
|
||||
const profileDefaults = requireQaRunProfile(profiles, profile);
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined),
|
||||
);
|
||||
return {
|
||||
profile,
|
||||
channel: normalizeQaChannel(payload.channel),
|
||||
channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver),
|
||||
evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode),
|
||||
providerMode,
|
||||
primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)),
|
||||
alternateModel: normalizeModel(
|
||||
@@ -116,16 +242,226 @@ export function normalizeQaRunSelection(
|
||||
defaultQaModelForMode(providerMode, true),
|
||||
),
|
||||
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
|
||||
runtimePair: normalizeQaRuntimePair(payload.runtimePair),
|
||||
runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane),
|
||||
scenarioIds: normalizeScenarioIds(payload.scenarioIds, scenarios),
|
||||
};
|
||||
}
|
||||
|
||||
export function createIdleQaRunnerSnapshot(scenarios: QaSeedScenario[]): QaLabRunnerSnapshot {
|
||||
function effectiveChannelForScenario(params: {
|
||||
scenario: QaSeedScenarioWithSource;
|
||||
selection: QaLabRunSelection;
|
||||
defaultChannel?: string;
|
||||
}): string | null {
|
||||
// Catalog parsing defaults flow executions to kind="flow"; plans never consume raw YAML.
|
||||
if (params.scenario.execution.kind !== "flow") {
|
||||
return null;
|
||||
}
|
||||
const fallbackChannel =
|
||||
params.defaultChannel ?? params.selection.channel ?? params.scenario.execution.channel;
|
||||
if (!fallbackChannel) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
resolveQaSuiteScenarioChannels({
|
||||
defaultChannel: fallbackChannel,
|
||||
explicitChannel:
|
||||
params.selection.channelDriver === "qa-channel" ? null : params.selection.channel,
|
||||
scenarios: [params.scenario],
|
||||
})[0] ?? null
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveQaLabRunPlan(params: {
|
||||
selection: QaLabRunSelection;
|
||||
scenarios: QaSeedScenarioWithSource[];
|
||||
scorecardReport: QaScorecardTaxonomyReport;
|
||||
defaultChannel?: string;
|
||||
supportsChannel?: (channel: string) => boolean;
|
||||
}): QaLabResolvedRunPlan {
|
||||
const { selection } = params;
|
||||
const explicitScenarioSelection = selection.scenarioIds !== null;
|
||||
const membership = resolveQaRunProfileMembership(
|
||||
{
|
||||
profile: selection.profile,
|
||||
scenarioIds: selection.scenarioIds ?? undefined,
|
||||
},
|
||||
{ scenarios: params.scenarios, scorecardReport: params.scorecardReport },
|
||||
);
|
||||
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
|
||||
const exclusions: QaLabResolvedRunPlan["exclusions"] = membership.excludedScenarioIds.map(
|
||||
(scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
return {
|
||||
scenarioId,
|
||||
executionKind: scenario ? scenario.execution.kind : "flow",
|
||||
reasons: [`not a member of profile ${selection.profile}`],
|
||||
};
|
||||
},
|
||||
);
|
||||
let laneSelection: ReturnType<typeof resolveQaRuntimePairLaneScenarioIds>;
|
||||
const errors: string[] = [];
|
||||
try {
|
||||
laneSelection = resolveQaRuntimePairLaneScenarioIds({
|
||||
channel: selection.channel,
|
||||
channelDriver: selection.channelDriver,
|
||||
defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined,
|
||||
primaryModel: selection.primaryModel,
|
||||
providerMode: selection.providerMode,
|
||||
scenarioIds: selection.runtimePairLane
|
||||
? []
|
||||
: membership.selectedScenarios.map((scenario) => scenario.id),
|
||||
scenarios: membership.profileScenarios,
|
||||
runtimePairLanes: selection.runtimePairLane ? [selection.runtimePairLane] : [],
|
||||
runtimePair: selection.runtimePair !== null,
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : String(error));
|
||||
laneSelection = {
|
||||
scenarioIds: [],
|
||||
excludedLaneScenarios: [],
|
||||
excludedNonFlowScenarios: [],
|
||||
};
|
||||
}
|
||||
exclusions.push(
|
||||
...laneSelection.excludedLaneScenarios.map((scenario) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons: ["does not match the selected provider/model/channel lane"],
|
||||
})),
|
||||
...laneSelection.excludedNonFlowScenarios.map((scenario) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons: ["runtimePair requires execution.kind=flow"],
|
||||
})),
|
||||
);
|
||||
if (selection.runtimePairLane && explicitScenarioSelection) {
|
||||
const laneScenarioIds = new Set(laneSelection.scenarioIds);
|
||||
const alreadyExcludedIds = new Set(exclusions.map((exclusion) => exclusion.scenarioId));
|
||||
exclusions.push(
|
||||
...membership.selectedScenarios
|
||||
.filter(
|
||||
(scenario) => !laneScenarioIds.has(scenario.id) && !alreadyExcludedIds.has(scenario.id),
|
||||
)
|
||||
.map((scenario) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons: [`runtimePairLane=${selection.runtimePairLane}`],
|
||||
})),
|
||||
);
|
||||
laneSelection = {
|
||||
...laneSelection,
|
||||
scenarioIds: membership.selectedScenarios
|
||||
.filter((scenario) => laneScenarioIds.has(scenario.id))
|
||||
.map((scenario) => scenario.id),
|
||||
};
|
||||
}
|
||||
const laneScenarios = laneSelection.scenarioIds.flatMap((scenarioId) => {
|
||||
const scenario = scenarioById.get(scenarioId);
|
||||
return scenario ? [scenario] : [];
|
||||
});
|
||||
const profileExecution = resolveQaRunProfileExecutionSelection({
|
||||
scenarios: laneScenarios,
|
||||
providerMode: selection.providerMode,
|
||||
primaryModel: selection.primaryModel,
|
||||
channelDriver: selection.channelDriver,
|
||||
channel: selection.channel,
|
||||
defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined,
|
||||
});
|
||||
exclusions.push(
|
||||
...profileExecution.excludedScenarios.map(({ scenario, reasons }) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons,
|
||||
})),
|
||||
);
|
||||
const runtimePairSupport = selection.runtimePair
|
||||
? resolveQaRuntimePairScenarioSupport(profileExecution.selectedScenarios)
|
||||
: { selectedScenarios: profileExecution.selectedScenarios, excludedScenarios: [] };
|
||||
exclusions.push(
|
||||
...runtimePairSupport.excludedScenarios.map((scenario) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons: ["runtimePair requires execution.kind=flow"],
|
||||
})),
|
||||
);
|
||||
let selectedScenarios = runtimePairSupport.selectedScenarios;
|
||||
if (selection.channelDriver !== "qa-channel" && params.supportsChannel) {
|
||||
const unsupportedChannelScenarios = selectedScenarios.flatMap((scenario) => {
|
||||
const channel = effectiveChannelForScenario({
|
||||
scenario,
|
||||
selection,
|
||||
defaultChannel: params.defaultChannel,
|
||||
});
|
||||
if (scenario.execution.kind !== "flow") {
|
||||
return [];
|
||||
}
|
||||
return channel !== null && !params.supportsChannel?.(channel) ? [{ scenario, channel }] : [];
|
||||
});
|
||||
const unsupportedIds = new Set(unsupportedChannelScenarios.map(({ scenario }) => scenario.id));
|
||||
exclusions.push(
|
||||
...unsupportedChannelScenarios.map(({ scenario, channel }) => ({
|
||||
scenarioId: scenario.id,
|
||||
executionKind: scenario.execution.kind,
|
||||
reasons: [`unsupported ${selection.channelDriver} channel=${channel}`],
|
||||
})),
|
||||
);
|
||||
selectedScenarios = selectedScenarios.filter((scenario) => !unsupportedIds.has(scenario.id));
|
||||
}
|
||||
if (membership.categories.length === 0) {
|
||||
errors.push(`QA run profile ${selection.profile} did not resolve any taxonomy categories.`);
|
||||
}
|
||||
if (selection.channel && selection.channelDriver === "qa-channel") {
|
||||
errors.push("An execution channel requires channelDriver=crabline or channelDriver=live.");
|
||||
}
|
||||
const explicitScenarioIds = new Set(selection.scenarioIds ?? []);
|
||||
const explicitExclusions = exclusions.filter((exclusion) =>
|
||||
explicitScenarioIds.has(exclusion.scenarioId),
|
||||
);
|
||||
if (explicitScenarioSelection && explicitExclusions.length > 0) {
|
||||
errors.push(
|
||||
`Explicit QA scenario selection is not runnable: ${explicitExclusions
|
||||
.map((exclusion) => `${exclusion.scenarioId} (${exclusion.reasons.join(", ")})`)
|
||||
.join("; ")}.`,
|
||||
);
|
||||
}
|
||||
if (selectedScenarios.length === 0) {
|
||||
errors.push("QA run plan selected no runnable scenarios.");
|
||||
}
|
||||
const executionKinds = uniqueStrings(
|
||||
selectedScenarios.map((scenario) => scenario.execution.kind),
|
||||
) as QaLabExecutionKind[];
|
||||
return {
|
||||
status: errors.length > 0 ? "invalid" : "ready",
|
||||
profile: selection.profile,
|
||||
explicitScenarioSelection,
|
||||
selectedScenarios: selectedScenarios.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
executionKind: scenario.execution.kind,
|
||||
declaredChannel: scenario.execution.channel ?? null,
|
||||
effectiveChannel: effectiveChannelForScenario({
|
||||
scenario,
|
||||
selection,
|
||||
defaultChannel: params.defaultChannel,
|
||||
}),
|
||||
})),
|
||||
executionKinds,
|
||||
exclusions,
|
||||
errors: uniqueStrings(errors),
|
||||
};
|
||||
}
|
||||
|
||||
export function createIdleQaRunnerSnapshot(
|
||||
profiles: readonly QaLabRunProfileOption[],
|
||||
plan: QaLabResolvedRunPlan | null = null,
|
||||
): QaLabRunnerSnapshot {
|
||||
return {
|
||||
status: "idle",
|
||||
selection: createDefaultQaRunSelection(scenarios, {
|
||||
selection: createDefaultQaRunSelection(profiles, {
|
||||
resolveDefaultModel: defaultStaticModelForMode,
|
||||
}),
|
||||
plan,
|
||||
artifacts: null,
|
||||
error: null,
|
||||
};
|
||||
|
||||
91
extensions/qa-lab/src/runtime-pair-lane-selection.ts
Normal file
91
extensions/qa-lab/src/runtime-pair-lane-selection.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
// Qa Lab plugin module owns canonical runtime-pair-lane scenario selection.
|
||||
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import {
|
||||
readQaScenarioPack,
|
||||
type QaRuntimePairLane,
|
||||
type QaSeedScenarioWithSource,
|
||||
} from "./scenario-catalog.js";
|
||||
import { scenarioMatchesQaProviderLane } from "./scenario-lane.js";
|
||||
import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js";
|
||||
|
||||
export function resolveQaRuntimePairScenarioSupport(
|
||||
scenarios: readonly QaSeedScenarioWithSource[],
|
||||
) {
|
||||
return {
|
||||
selectedScenarios: scenarios.filter((scenario) => scenario.execution.kind === "flow"),
|
||||
excludedScenarios: scenarios.filter((scenario) => scenario.execution.kind !== "flow"),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveQaRuntimePairLaneScenarioIds(params: {
|
||||
channel?: string | null;
|
||||
channelDriver?: QaScorecardChannelDriver | null;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
defaultChannel?: string;
|
||||
primaryModel: string;
|
||||
providerMode: QaProviderMode;
|
||||
scenarioIds: string[];
|
||||
scenarios?: QaSeedScenarioWithSource[];
|
||||
runtimePairLanes: readonly QaRuntimePairLane[];
|
||||
runtimePair: boolean;
|
||||
}): {
|
||||
scenarioIds: string[];
|
||||
excludedLaneScenarios: QaSeedScenarioWithSource[];
|
||||
excludedNonFlowScenarios: QaSeedScenarioWithSource[];
|
||||
} {
|
||||
if (params.runtimePairLanes.length === 0) {
|
||||
return {
|
||||
scenarioIds: params.scenarioIds,
|
||||
excludedLaneScenarios: [],
|
||||
excludedNonFlowScenarios: [],
|
||||
};
|
||||
}
|
||||
const laneSet = new Set(params.runtimePairLanes);
|
||||
const matchingScenarios = (params.scenarios ?? readQaScenarioPack().scenarios).filter(
|
||||
(scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane),
|
||||
);
|
||||
if (matchingScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
const compatibleScenarios = params.runtimePair
|
||||
? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow")
|
||||
: matchingScenarios;
|
||||
const laneCompatibleScenarios = compatibleScenarios.filter((scenario) =>
|
||||
scenarioMatchesQaProviderLane({
|
||||
scenario,
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
channelDriver: params.channelDriver,
|
||||
channel: params.channel ?? scenario.execution.channel ?? params.defaultChannel,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
}),
|
||||
);
|
||||
const excludedLaneScenarios = compatibleScenarios.filter(
|
||||
(scenario) => !laneCompatibleScenarios.includes(scenario),
|
||||
);
|
||||
const excludedNonFlowScenarios = params.runtimePair
|
||||
? matchingScenarios.filter((scenario) => scenario.execution.kind !== "flow")
|
||||
: [];
|
||||
if (compatibleScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}.`,
|
||||
);
|
||||
}
|
||||
if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) {
|
||||
throw new Error(
|
||||
`--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
scenarioIds: uniqueStrings([
|
||||
...params.scenarioIds,
|
||||
...laneCompatibleScenarios.map((scenario) => scenario.id),
|
||||
]),
|
||||
excludedLaneScenarios,
|
||||
excludedNonFlowScenarios,
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { QaLabServerHandle } from "./lab-server.types.js";
|
||||
import type { QaSuiteScenarioResult } from "./suite.js";
|
||||
import type {
|
||||
QaTestFileScenario,
|
||||
@@ -542,7 +543,19 @@ describe("qa suite runtime launcher", () => {
|
||||
|
||||
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-launch-");
|
||||
const setScenarioRun = vi.fn<QaLabServerHandle["setScenarioRun"]>();
|
||||
const lab = {
|
||||
baseUrl: "http://127.0.0.1:43124",
|
||||
listenUrl: "http://127.0.0.1:43124",
|
||||
runSelfCheck: vi.fn(),
|
||||
setControlUi: vi.fn(),
|
||||
setLatestReport: vi.fn(),
|
||||
setScenarioRun,
|
||||
state: {} as QaLabServerHandle["state"],
|
||||
stop: vi.fn(),
|
||||
} satisfies QaLabServerHandle;
|
||||
const result = await runQaSuite({
|
||||
lab,
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/scenario-test",
|
||||
scenarioIds: ["control-ui-chat-flow-playwright"],
|
||||
@@ -582,6 +595,14 @@ describe("qa suite runtime launcher", () => {
|
||||
kind: scenario.execution.kind,
|
||||
})),
|
||||
).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]);
|
||||
expect(setScenarioRun).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
status: "completed",
|
||||
scenarios: [
|
||||
expect.objectContaining({ id: "control-ui-chat-flow-playwright", status: "pass" }),
|
||||
],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("serializes test-file runner partitions in one checkout", async () => {
|
||||
@@ -699,6 +720,96 @@ describe("qa suite runtime launcher", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("aggregates mixed-kind progress through the parent lab", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-mixed-progress-");
|
||||
const scenarioRuns: Array<Parameters<QaLabServerHandle["setScenarioRun"]>[0]> = [];
|
||||
const lab = {
|
||||
baseUrl: "http://127.0.0.1:43124",
|
||||
listenUrl: "http://127.0.0.1:43124",
|
||||
runSelfCheck: vi.fn(),
|
||||
setControlUi: vi.fn(),
|
||||
setLatestReport: vi.fn(),
|
||||
setScenarioRun: vi.fn((run) => scenarioRuns.push(run)),
|
||||
state: {} as QaLabServerHandle["state"],
|
||||
stop: vi.fn(),
|
||||
} satisfies QaLabServerHandle;
|
||||
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
|
||||
if (!defaultFlowImplementation) {
|
||||
throw new Error("expected default QA flow suite mock implementation");
|
||||
}
|
||||
runQaFlowSuite.mockImplementationOnce(async (params) => {
|
||||
params?.lab?.setScenarioRun({
|
||||
kind: "suite",
|
||||
status: "running",
|
||||
startedAt: "2026-07-23T00:00:00.000Z",
|
||||
scenarios: [
|
||||
{ id: "channel-chat-baseline", name: "channel-chat-baseline", status: "running" },
|
||||
],
|
||||
});
|
||||
params?.lab?.setScenarioRun({
|
||||
kind: "suite",
|
||||
status: "running",
|
||||
startedAt: "2026-07-23T00:00:00.000Z",
|
||||
scenarios: [
|
||||
{ id: "not-a-selected-scenario", name: "channel-chat-baseline", status: "fail" },
|
||||
],
|
||||
});
|
||||
const result = await defaultFlowImplementation(params);
|
||||
params?.lab?.setScenarioRun({
|
||||
kind: "suite",
|
||||
status: "completed",
|
||||
startedAt: "2026-07-23T00:00:00.000Z",
|
||||
finishedAt: "2026-07-23T00:00:01.000Z",
|
||||
scenarios: [{ id: "channel-chat-baseline", name: "channel-chat-baseline", status: "pass" }],
|
||||
});
|
||||
return result;
|
||||
});
|
||||
|
||||
await runQaSuite({
|
||||
lab,
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mixed-progress",
|
||||
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
|
||||
});
|
||||
|
||||
expect(
|
||||
scenarioRuns.some(
|
||||
(run) =>
|
||||
run?.status === "running" &&
|
||||
run.scenarios.some(
|
||||
(scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "running",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
scenarioRuns.some((run) =>
|
||||
run?.scenarios.some(
|
||||
(scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "fail",
|
||||
),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
scenarioRuns.some(
|
||||
(run) =>
|
||||
run?.status === "running" &&
|
||||
run.scenarios.some(
|
||||
(scenario) =>
|
||||
scenario.id === "control-ui-chat-flow-playwright" && scenario.status === "running",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(scenarioRuns.at(-1)).toMatchObject({
|
||||
status: "completed",
|
||||
scenarios: [
|
||||
{ id: "channel-chat-baseline", status: "pass" },
|
||||
{ id: "control-ui-chat-flow-playwright", status: "pass" },
|
||||
],
|
||||
});
|
||||
expect(lab.setLatestReport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ outputPath: expect.stringMatching(/qa-suite-report\.md$/u) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps channel-driver unified flow partitions serial by default", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-crabline-serial-");
|
||||
await runQaSuite({
|
||||
@@ -870,6 +981,17 @@ describe("qa suite runtime launcher", () => {
|
||||
|
||||
it("stops unified suite partitions after the first failed flow scenario", async () => {
|
||||
const repoRoot = await makeTempRepo("qa-suite-fail-fast-flow-");
|
||||
const scenarioRuns: Array<Parameters<QaLabServerHandle["setScenarioRun"]>[0]> = [];
|
||||
const lab = {
|
||||
baseUrl: "http://127.0.0.1:43124",
|
||||
listenUrl: "http://127.0.0.1:43124",
|
||||
runSelfCheck: vi.fn(),
|
||||
setControlUi: vi.fn(),
|
||||
setLatestReport: vi.fn(),
|
||||
setScenarioRun: vi.fn((run) => scenarioRuns.push(run)),
|
||||
state: {} as QaLabServerHandle["state"],
|
||||
stop: vi.fn(),
|
||||
} satisfies QaLabServerHandle;
|
||||
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
|
||||
if (!defaultFlowImplementation) {
|
||||
throw new Error("expected default QA flow suite mock implementation");
|
||||
@@ -888,6 +1010,7 @@ describe("qa suite runtime launcher", () => {
|
||||
});
|
||||
|
||||
const result = await runQaSuite({
|
||||
lab,
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/fail-fast-flow",
|
||||
concurrency: 8,
|
||||
@@ -928,6 +1051,15 @@ describe("qa suite runtime launcher", () => {
|
||||
"docker-npm-onboard-channel-agent",
|
||||
]);
|
||||
expect(summary.scenarios).toMatchObject([{ name: "dm-chat-baseline", status: "fail" }]);
|
||||
expect(scenarioRuns.at(-1)).toMatchObject({
|
||||
status: "completed",
|
||||
scenarios: [
|
||||
{ id: "dm-chat-baseline", status: "fail" },
|
||||
{ id: "group-visible-reply-tool", status: "pending" },
|
||||
{ id: "control-ui-chat-flow-playwright", status: "pending" },
|
||||
{ id: "docker-npm-onboard-channel-agent", status: "pending" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("stops pending flow and script partitions after a native scenario fails", async () => {
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
resolveQaSuiteWorkerStartStaggerMs,
|
||||
scenarioRequiresIsolatedQaSuiteWorker,
|
||||
} from "./suite-planning.js";
|
||||
import { createQaSuiteProgressController } from "./suite-progress.js";
|
||||
import {
|
||||
buildQaSuiteSummaryJson,
|
||||
type QaSuiteResult,
|
||||
@@ -604,6 +605,14 @@ async function runUnifiedQaSuite(params: {
|
||||
const startedAt = new Date();
|
||||
const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd());
|
||||
const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir);
|
||||
const progress = params.runParams?.lab
|
||||
? createQaSuiteProgressController({
|
||||
lab: params.runParams.lab,
|
||||
scenarios: params.plan.scenarios,
|
||||
startedAt: startedAt.toISOString(),
|
||||
})
|
||||
: undefined;
|
||||
progress?.start();
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params.runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE,
|
||||
);
|
||||
@@ -774,6 +783,13 @@ async function runUnifiedQaSuite(params: {
|
||||
}
|
||||
const result = await runFlowSuite({
|
||||
...params.runParams,
|
||||
...(progress
|
||||
? {
|
||||
lab: progress.createPartitionLab(
|
||||
partition.scenarios.map((scenario) => scenario.id),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
outputDir: partitionName
|
||||
? flowSuitePartitionOutputDir(outputDir, partitionName)
|
||||
: suitePartitionOutputDir(outputDir, "flow"),
|
||||
@@ -852,6 +868,11 @@ async function runUnifiedQaSuite(params: {
|
||||
const testFileScenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
|
||||
const testFileStartedScenarioIds: string[] = [];
|
||||
for (const [kind, testFileScenarios] of scenariosByKind) {
|
||||
progress?.markRunning(
|
||||
(failFast ? testFileScenarios.slice(0, 1) : testFileScenarios).map(
|
||||
(scenario) => scenario.id,
|
||||
),
|
||||
);
|
||||
const result = await runQaTestFileSuiteFromRuntime({
|
||||
runParams: {
|
||||
...params.runParams,
|
||||
@@ -869,12 +890,12 @@ async function runUnifiedQaSuite(params: {
|
||||
repoRoot,
|
||||
}),
|
||||
);
|
||||
testFileScenarioResults.push(
|
||||
...result.results.map((scenarioResult) => ({
|
||||
scenarioId: scenarioResult.scenario.id,
|
||||
result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot),
|
||||
})),
|
||||
);
|
||||
const scenarioResults = result.results.map((scenarioResult) => ({
|
||||
scenarioId: scenarioResult.scenario.id,
|
||||
result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot),
|
||||
}));
|
||||
testFileScenarioResults.push(...scenarioResults);
|
||||
progress?.recordResults(scenarioResults);
|
||||
const resultsByScenarioId = new Map(
|
||||
result.results.map((scenarioResult) => [scenarioResult.scenario.id, scenarioResult]),
|
||||
);
|
||||
@@ -1049,7 +1070,7 @@ async function runUnifiedQaSuite(params: {
|
||||
} satisfies QaSuiteScenarioResult,
|
||||
];
|
||||
});
|
||||
return await writeUnifiedQaSuiteArtifacts({
|
||||
const unifiedResult = await writeUnifiedQaSuiteArtifacts({
|
||||
alternateModel,
|
||||
channelDriver: params.runParams?.channelDriver,
|
||||
concurrency,
|
||||
@@ -1064,6 +1085,33 @@ async function runUnifiedQaSuite(params: {
|
||||
scenarios,
|
||||
startedAt,
|
||||
});
|
||||
const progressResults = params.plan.scenarios.flatMap((scenario) => {
|
||||
const result = scenarioResultsById.get(scenario.id);
|
||||
if (result) {
|
||||
return [{ scenarioId: scenario.id, result }];
|
||||
}
|
||||
if (failFast && !startedScenarioIds.has(scenario.id)) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
scenarioId: scenario.id,
|
||||
result: {
|
||||
name: scenario.title,
|
||||
status: "fail" as const,
|
||||
details: "suite partition returned no scenario result",
|
||||
steps: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
});
|
||||
progress?.complete(progressResults, finishedAt.toISOString());
|
||||
params.runParams?.lab?.setLatestReport({
|
||||
outputPath: unifiedResult.reportPath,
|
||||
markdown: unifiedResult.report,
|
||||
generatedAt: finishedAt.toISOString(),
|
||||
});
|
||||
return unifiedResult;
|
||||
}
|
||||
|
||||
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
|
||||
|
||||
126
extensions/qa-lab/src/suite-progress.ts
Normal file
126
extensions/qa-lab/src/suite-progress.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import type {
|
||||
QaLabScenarioOutcome,
|
||||
QaLabScenarioRun,
|
||||
QaLabServerHandle,
|
||||
} from "./lab-server.types.js";
|
||||
|
||||
type QaSuiteProgressScenario = {
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
type QaSuiteProgressResult = {
|
||||
scenarioId: string;
|
||||
result: {
|
||||
name: string;
|
||||
status: "pass" | "fail" | "skip";
|
||||
steps: QaLabScenarioOutcome["steps"];
|
||||
details?: string;
|
||||
};
|
||||
};
|
||||
|
||||
function cloneOutcome(outcome: QaLabScenarioOutcome): QaLabScenarioOutcome {
|
||||
return {
|
||||
...outcome,
|
||||
...(outcome.steps ? { steps: outcome.steps.map((step) => ({ ...step })) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function createQaSuiteProgressController(params: {
|
||||
lab: QaLabServerHandle;
|
||||
scenarios: readonly QaSuiteProgressScenario[];
|
||||
startedAt: string;
|
||||
}) {
|
||||
const outcomes = new Map<string, QaLabScenarioOutcome>(
|
||||
params.scenarios.map((scenario) => [
|
||||
scenario.id,
|
||||
{
|
||||
id: scenario.id,
|
||||
name: scenario.title,
|
||||
status: "pending" as const,
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const emit = (status: QaLabScenarioRun["status"], finishedAt?: string) => {
|
||||
params.lab.setScenarioRun({
|
||||
kind: "suite",
|
||||
status,
|
||||
startedAt: params.startedAt,
|
||||
...(finishedAt ? { finishedAt } : {}),
|
||||
scenarios: params.scenarios.map((scenario) => cloneOutcome(outcomes.get(scenario.id)!)),
|
||||
});
|
||||
};
|
||||
|
||||
const updateResult = (entry: QaSuiteProgressResult, finishedAt?: string) => {
|
||||
const current = outcomes.get(entry.scenarioId);
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
outcomes.set(entry.scenarioId, {
|
||||
...current,
|
||||
name: entry.result.name,
|
||||
status: entry.result.status,
|
||||
...(entry.result.details ? { details: entry.result.details } : {}),
|
||||
...(entry.result.steps ? { steps: entry.result.steps } : {}),
|
||||
...(finishedAt ? { finishedAt } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
start() {
|
||||
emit("running");
|
||||
},
|
||||
markRunning(scenarioIds: readonly string[]) {
|
||||
const startedAt = new Date().toISOString();
|
||||
for (const scenarioId of scenarioIds) {
|
||||
const current = outcomes.get(scenarioId);
|
||||
if (!current || current.status !== "pending") {
|
||||
continue;
|
||||
}
|
||||
outcomes.set(scenarioId, { ...current, status: "running", startedAt });
|
||||
}
|
||||
emit("running");
|
||||
},
|
||||
recordResults(entries: readonly QaSuiteProgressResult[]) {
|
||||
const finishedAt = new Date().toISOString();
|
||||
for (const entry of entries) {
|
||||
updateResult(entry, finishedAt);
|
||||
}
|
||||
emit("running");
|
||||
},
|
||||
createPartitionLab(scenarioIds: readonly string[]): QaLabServerHandle {
|
||||
const partitionIds = new Set(scenarioIds);
|
||||
return {
|
||||
...params.lab,
|
||||
setScenarioRun(next) {
|
||||
if (!next) {
|
||||
return;
|
||||
}
|
||||
for (const nextOutcome of next.scenarios) {
|
||||
if (!partitionIds.has(nextOutcome.id)) {
|
||||
continue;
|
||||
}
|
||||
const current = outcomes.get(nextOutcome.id);
|
||||
if (!current) {
|
||||
continue;
|
||||
}
|
||||
outcomes.set(nextOutcome.id, {
|
||||
...current,
|
||||
...nextOutcome,
|
||||
});
|
||||
}
|
||||
emit("running");
|
||||
},
|
||||
// Child partition reports are incomplete. The unified owner publishes one aggregate.
|
||||
setLatestReport() {},
|
||||
};
|
||||
},
|
||||
complete(entries: readonly QaSuiteProgressResult[], finishedAt: string) {
|
||||
for (const entry of entries) {
|
||||
updateResult(entry, finishedAt);
|
||||
}
|
||||
emit("completed", finishedAt);
|
||||
},
|
||||
};
|
||||
}
|
||||
424
extensions/qa-lab/web/src/app.browser.test.ts
Normal file
424
extensions/qa-lab/web/src/app.browser.test.ts
Normal file
@@ -0,0 +1,424 @@
|
||||
/* @vitest-environment jsdom */
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Bootstrap, RunnerSelection } from "./ui-types.js";
|
||||
|
||||
const httpMock = vi.hoisted(() => {
|
||||
class QaLabHttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly payload: unknown,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
return {
|
||||
getJson: vi.fn(),
|
||||
getJsonNoStore: vi.fn(),
|
||||
postJson: vi.fn(),
|
||||
QaLabHttpError,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./http.js", () => httpMock);
|
||||
|
||||
import { createQaLabApp } from "./app.js";
|
||||
|
||||
const scenarios: Bootstrap["scenarios"] = [
|
||||
{
|
||||
id: "dm-chat-baseline",
|
||||
title: "DM baseline",
|
||||
surface: "dm",
|
||||
objective: "test DM",
|
||||
successCriteria: ["reply"],
|
||||
execution: { kind: "flow" },
|
||||
},
|
||||
{
|
||||
id: "browser-talk-start-stop",
|
||||
title: "Browser Talk start-stop",
|
||||
surface: "control-ui",
|
||||
objective: "test browser Talk",
|
||||
successCriteria: ["playwright pass"],
|
||||
execution: { kind: "playwright" },
|
||||
},
|
||||
];
|
||||
|
||||
function createBootstrap(selection: RunnerSelection): Bootstrap {
|
||||
const selectedScenarioIds = selection.scenarioIds ?? scenarios.map((scenario) => scenario.id);
|
||||
return {
|
||||
baseUrl: "http://127.0.0.1:43124",
|
||||
controlUiEmbeddedUrl: null,
|
||||
controlUiUrl: null,
|
||||
defaults: {
|
||||
conversationId: "qa-operator",
|
||||
conversationKind: "direct",
|
||||
senderId: "qa-operator",
|
||||
senderName: "QA Operator",
|
||||
},
|
||||
kickoffTask: "Run QA",
|
||||
latestReport: null,
|
||||
runner: {
|
||||
artifacts: null,
|
||||
error: null,
|
||||
plan: {
|
||||
errors: [],
|
||||
exclusions: [],
|
||||
executionKinds: ["flow", "playwright"],
|
||||
explicitScenarioSelection: selection.scenarioIds !== null,
|
||||
profile: selection.profile,
|
||||
selectedScenarios: scenarios
|
||||
.filter((scenario) => selectedScenarioIds.includes(scenario.id))
|
||||
.map((scenario) => ({
|
||||
declaredChannel: null,
|
||||
effectiveChannel: scenario.execution?.kind === "flow" ? "qa-channel" : null,
|
||||
executionKind: scenario.execution?.kind ?? "flow",
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
})),
|
||||
status: "ready",
|
||||
},
|
||||
selection,
|
||||
status: "idle",
|
||||
},
|
||||
runnerCatalog: {
|
||||
channels: ["matrix", "telegram"],
|
||||
profiles: [
|
||||
{ id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] },
|
||||
{ id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] },
|
||||
],
|
||||
status: "ready",
|
||||
real: [
|
||||
{
|
||||
input: "text",
|
||||
key: "openai/gpt-5.6-luna",
|
||||
name: "GPT-5.6 Luna",
|
||||
preferred: true,
|
||||
provider: "openai",
|
||||
},
|
||||
],
|
||||
},
|
||||
scenarios,
|
||||
};
|
||||
}
|
||||
|
||||
async function mountRunner(selection: RunnerSelection) {
|
||||
let bootstrap = createBootstrap(selection);
|
||||
httpMock.getJson.mockImplementation(async (url: string) => {
|
||||
if (url === "/api/bootstrap") {
|
||||
return bootstrap;
|
||||
}
|
||||
if (url === "/api/state") {
|
||||
return { conversations: [], events: [], messages: [], threads: [] };
|
||||
}
|
||||
if (url === "/api/report") {
|
||||
return { report: null };
|
||||
}
|
||||
if (url === "/api/outcomes") {
|
||||
return { run: null };
|
||||
}
|
||||
if (url === "/api/capture/sessions") {
|
||||
return { sessions: [] };
|
||||
}
|
||||
if (url === "/api/capture/startup-status") {
|
||||
return {
|
||||
status: {
|
||||
gateway: { label: "Gateway", ok: true, url: "http://127.0.0.1:18789" },
|
||||
proxy: { label: "Proxy", ok: true, url: "http://127.0.0.1:7799" },
|
||||
qaLab: { label: "QA Lab", ok: true, url: bootstrap.baseUrl },
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected GET ${url}`);
|
||||
});
|
||||
httpMock.getJsonNoStore.mockResolvedValue({ version: "test" });
|
||||
httpMock.postJson.mockImplementation(async (url: string, body: unknown) => {
|
||||
if (url !== "/api/scenario/suite") {
|
||||
throw new Error(`unexpected POST ${url}`);
|
||||
}
|
||||
const nextSelection = body as RunnerSelection;
|
||||
bootstrap = createBootstrap(nextSelection);
|
||||
return { runner: { selection: nextSelection } };
|
||||
});
|
||||
const root = document.createElement("div");
|
||||
document.body.append(root);
|
||||
await createQaLabApp(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
function selectValue(root: HTMLElement, selector: string, value: string) {
|
||||
const select = root.querySelector<HTMLSelectElement>(selector);
|
||||
if (!select) {
|
||||
throw new Error(`missing select ${selector}`);
|
||||
}
|
||||
select.value = value;
|
||||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
const styles = document.createElement("style");
|
||||
styles.dataset.qaLabTestStyles = "true";
|
||||
styles.textContent = readFileSync(
|
||||
path.join(process.cwd(), "extensions/qa-lab/web/src/styles.css"),
|
||||
"utf8",
|
||||
);
|
||||
document.head.append(styles);
|
||||
httpMock.getJson.mockReset();
|
||||
httpMock.getJsonNoStore.mockReset();
|
||||
httpMock.postJson.mockReset();
|
||||
const storage = new Map<string, string>();
|
||||
vi.stubGlobal("localStorage", {
|
||||
clear: () => storage.clear(),
|
||||
getItem: (key: string) => storage.get(key) ?? null,
|
||||
key: (index: number) => [...storage.keys()][index] ?? null,
|
||||
get length() {
|
||||
return storage.size;
|
||||
},
|
||||
removeItem: (key: string) => storage.delete(key),
|
||||
setItem: (key: string, value: string) => storage.set(key, value),
|
||||
});
|
||||
vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => {
|
||||
callback(0);
|
||||
return 1;
|
||||
});
|
||||
vi.stubGlobal("matchMedia", () => ({ matches: false }));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
document.body.replaceChildren();
|
||||
document.querySelector("style[data-qa-lab-test-styles]")?.remove();
|
||||
});
|
||||
|
||||
describe("QA Lab runner browser interactions", () => {
|
||||
it("keeps scenario rows from collapsing inside the scrolling list", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "qa-channel",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
const scroll = root.querySelector<HTMLElement>(".scenario-scroll");
|
||||
const row = root.querySelector<HTMLElement>(".scenario-item");
|
||||
expect(scroll).not.toBeNull();
|
||||
expect(row).not.toBeNull();
|
||||
expect(getComputedStyle(scroll!).overflowY).toBe("auto");
|
||||
expect(getComputedStyle(row!).flexShrink).toBe("0");
|
||||
});
|
||||
|
||||
it("submits live-provider and Crabline selections with non-flow scenarios", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
channel: null,
|
||||
channelDriver: "crabline",
|
||||
evidenceMode: "full",
|
||||
fastMode: true,
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "live-frontier",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-action='select-all-scenarios']")?.click();
|
||||
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
|
||||
|
||||
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
|
||||
expect(httpMock.postJson).toHaveBeenCalledWith(
|
||||
"/api/scenario/suite",
|
||||
expect.objectContaining({
|
||||
channelDriver: "crabline",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("changes to real channels without changing the mock provider lane", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "qa-channel",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
|
||||
selectValue(root, "#channel-driver", "live");
|
||||
selectValue(root, "#execution-channel", "telegram");
|
||||
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
|
||||
|
||||
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
|
||||
expect(httpMock.postJson).toHaveBeenCalledWith(
|
||||
"/api/scenario/suite",
|
||||
expect.objectContaining({
|
||||
channelDriver: "live",
|
||||
channel: "telegram",
|
||||
providerMode: "mock-openai",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("submits profile, evidence, runtime-pair, lane, and channel controls", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "live",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
|
||||
selectValue(root, "#run-profile", "smoke-ci");
|
||||
selectValue(root, "#execution-channel", "telegram");
|
||||
selectValue(root, "#evidence-mode", "slim");
|
||||
selectValue(root, "#runtime-pair", "openclaw,codex");
|
||||
selectValue(root, "#runtime-pair-lane", "core");
|
||||
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
|
||||
|
||||
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
|
||||
expect(httpMock.postJson).toHaveBeenCalledWith(
|
||||
"/api/scenario/suite",
|
||||
expect.objectContaining({
|
||||
profile: "smoke-ci",
|
||||
channel: "telegram",
|
||||
channelDriver: "crabline",
|
||||
evidenceMode: "slim",
|
||||
runtimePair: ["openclaw", "codex"],
|
||||
runtimePairLane: "core",
|
||||
scenarioIds: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("renders server-resolved exclusions and errors from a rejected launch", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "qa-channel",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
httpMock.postJson.mockRejectedValueOnce(
|
||||
new httpMock.QaLabHttpError("selection rejected", 400, {
|
||||
plan: {
|
||||
errors: ["Explicit QA scenario selection is not runnable."],
|
||||
exclusions: [
|
||||
{
|
||||
executionKind: "flow",
|
||||
reasons: ["channel=telegram"],
|
||||
scenarioId: "dm-chat-baseline",
|
||||
},
|
||||
],
|
||||
executionKinds: [],
|
||||
explicitScenarioSelection: true,
|
||||
profile: "all",
|
||||
selectedScenarios: [],
|
||||
status: "invalid",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
|
||||
|
||||
await vi.waitFor(() => expect(root.textContent).toContain("1 excluded"));
|
||||
expect(root.textContent).toContain("Explicit QA scenario selection is not runnable");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(root.textContent).toContain("1 excluded"));
|
||||
expect(root.textContent).toContain("Explicit QA scenario selection is not runnable");
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
|
||||
selectValue(root, "#evidence-mode", "slim");
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='run']")?.click();
|
||||
expect(root.textContent).not.toContain("Explicit QA scenario selection is not runnable");
|
||||
expect(root.textContent).not.toContain("Resolved plan:");
|
||||
});
|
||||
|
||||
it("starts a dirty profile override without reusing the previous resolved plan", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "qa-channel",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='config']")?.click();
|
||||
selectValue(root, "#run-profile", "smoke-ci");
|
||||
root.querySelector<HTMLButtonElement>("[data-sidebar-panel='scenarios']")?.click();
|
||||
root
|
||||
.querySelector<HTMLInputElement>("[data-scenario-toggle-id='browser-talk-start-stop']")
|
||||
?.click();
|
||||
root.querySelector<HTMLButtonElement>("[data-action='run-suite']")?.click();
|
||||
|
||||
await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1));
|
||||
expect(httpMock.postJson).toHaveBeenCalledWith(
|
||||
"/api/scenario/suite",
|
||||
expect.objectContaining({
|
||||
profile: "smoke-ci",
|
||||
scenarioIds: ["browser-talk-start-stop"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("disables launch when an explicit override becomes empty", async () => {
|
||||
const root = await mountRunner({
|
||||
alternateModel: "mock-openai/gpt-5.6-luna-alt",
|
||||
channel: null,
|
||||
channelDriver: "qa-channel",
|
||||
evidenceMode: "full",
|
||||
fastMode: false,
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
profile: "all",
|
||||
providerMode: "mock-openai",
|
||||
runtimePair: null,
|
||||
runtimePairLane: null,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
|
||||
root.querySelector<HTMLInputElement>("[data-scenario-toggle-id='dm-chat-baseline']")?.click();
|
||||
const runButton = root.querySelector<HTMLButtonElement>("[data-action='run-suite']");
|
||||
expect(runButton?.disabled).toBe(true);
|
||||
expect(runButton?.textContent).toContain("Run 0 scenarios");
|
||||
runButton?.click();
|
||||
expect(httpMock.postJson).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -2,13 +2,14 @@
|
||||
import { defaultQaModelForMode, isQaFastModeEnabled } from "../../model-selection.js";
|
||||
import { normalizeCaptureSavedView, normalizeCaptureSavedViews } from "./capture-saved-view.js";
|
||||
import { formatErrorMessage } from "./errors.js";
|
||||
import { getJson, getJsonNoStore, postJson } from "./http.js";
|
||||
import { getJson, getJsonNoStore, postJson, QaLabHttpError } from "./http.js";
|
||||
import { conversationSelectionKey, findConversationBySelectionKey } from "./ui-conversation-key.js";
|
||||
import {
|
||||
type Bootstrap,
|
||||
type EvidenceEnvelope,
|
||||
type OutcomesEnvelope,
|
||||
type ReportEnvelope,
|
||||
type RunnerResolvedPlan,
|
||||
type RunnerSelection,
|
||||
type Snapshot,
|
||||
type TabId,
|
||||
@@ -84,6 +85,14 @@ function defaultModelsForProviderMode(
|
||||
};
|
||||
}
|
||||
|
||||
function cloneRunnerSelection(selection: RunnerSelection): RunnerSelection {
|
||||
return {
|
||||
...selection,
|
||||
runtimePair: selection.runtimePair ? [...selection.runtimePair] : null,
|
||||
scenarioIds: selection.scenarioIds ? [...selection.scenarioIds] : null,
|
||||
};
|
||||
}
|
||||
|
||||
function detectTheme(): "light" | "dark" {
|
||||
const stored = localStorage.getItem("qa-lab-theme");
|
||||
if (stored === "light" || stored === "dark") {
|
||||
@@ -207,6 +216,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
activeTab: initialUrl.pathname === "/evidence" || initialEvidencePath ? "evidence" : "chat",
|
||||
runnerDraft: null,
|
||||
runnerDraftDirty: false,
|
||||
runnerPlanOverride: null,
|
||||
composer: {
|
||||
conversationKind: "direct",
|
||||
conversationId: "alice",
|
||||
@@ -250,6 +260,9 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
ra: state.bootstrap?.runner.startedAt,
|
||||
rf: state.bootstrap?.runner.finishedAt,
|
||||
re: state.bootstrap?.runner.error,
|
||||
rpo: state.runnerPlanOverride
|
||||
? `${state.runnerPlanOverride.status}:${state.runnerPlanOverride.selectedScenarios.length}:${state.runnerPlanOverride.exclusions.length}:${state.runnerPlanOverride.errors.join("|")}`
|
||||
: null,
|
||||
ss: state.scenarioRun?.status,
|
||||
sc: state.scenarioRun?.counts,
|
||||
so: state.scenarioRun?.scenarios.map((o) => o.status).join(","),
|
||||
@@ -341,10 +354,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
state.evidencePathDraft = bootstrap.runner.artifacts.evidencePath;
|
||||
}
|
||||
if (!state.runnerDraft || !state.runnerDraftDirty) {
|
||||
state.runnerDraft = {
|
||||
...bootstrap.runner.selection,
|
||||
scenarioIds: [...bootstrap.runner.selection.scenarioIds],
|
||||
};
|
||||
state.runnerDraft = cloneRunnerSelection(bootstrap.runner.selection);
|
||||
state.runnerDraftDirty = false;
|
||||
}
|
||||
if (!state.selectedConversationKey) {
|
||||
@@ -497,13 +507,14 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
function updateRunnerDraft(mutator: (draft: RunnerSelection) => RunnerSelection) {
|
||||
const fallback = state.bootstrap?.runner.selection;
|
||||
if (!state.runnerDraft && fallback) {
|
||||
state.runnerDraft = { ...fallback, scenarioIds: [...fallback.scenarioIds] };
|
||||
state.runnerDraft = cloneRunnerSelection(fallback);
|
||||
}
|
||||
if (!state.runnerDraft) {
|
||||
return;
|
||||
}
|
||||
state.runnerDraft = mutator(state.runnerDraft);
|
||||
state.runnerDraftDirty = true;
|
||||
state.runnerPlanOverride = null;
|
||||
render();
|
||||
}
|
||||
|
||||
@@ -610,20 +621,32 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
const result = await postJson<{ runner: { selection: RunnerSelection } }>(
|
||||
"/api/scenario/suite",
|
||||
{
|
||||
profile: state.runnerDraft.profile,
|
||||
channel: state.runnerDraft.channel,
|
||||
channelDriver: state.runnerDraft.channelDriver,
|
||||
evidenceMode: state.runnerDraft.evidenceMode,
|
||||
providerMode: state.runnerDraft.providerMode,
|
||||
primaryModel: state.runnerDraft.primaryModel,
|
||||
alternateModel: state.runnerDraft.alternateModel,
|
||||
fastMode: state.runnerDraft.fastMode,
|
||||
runtimePair: state.runnerDraft.runtimePair,
|
||||
runtimePairLane: state.runnerDraft.runtimePairLane,
|
||||
scenarioIds: state.runnerDraft.scenarioIds,
|
||||
},
|
||||
);
|
||||
state.runnerDraft = {
|
||||
...result.runner.selection,
|
||||
scenarioIds: [...result.runner.selection.scenarioIds],
|
||||
};
|
||||
state.runnerDraft = cloneRunnerSelection(result.runner.selection);
|
||||
state.runnerDraftDirty = false;
|
||||
state.runnerPlanOverride = null;
|
||||
state.activeTab = "chat";
|
||||
await refresh();
|
||||
} catch (error) {
|
||||
if (error instanceof QaLabHttpError) {
|
||||
const plan = (error.payload as { plan?: RunnerResolvedPlan } | null)?.plan;
|
||||
if (plan) {
|
||||
state.runnerPlanOverride = plan;
|
||||
state.sidebarPanel = "run";
|
||||
}
|
||||
}
|
||||
state.error = formatErrorMessage(error);
|
||||
render();
|
||||
} finally {
|
||||
@@ -904,7 +927,7 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
root
|
||||
.querySelector<HTMLElement>("[data-action='clear-scenarios']")
|
||||
?.addEventListener("click", () => {
|
||||
updateRunnerDraft((d) => ({ ...d, scenarioIds: [] }));
|
||||
updateRunnerDraft((d) => ({ ...d, scenarioIds: null }));
|
||||
});
|
||||
|
||||
/* Scenario toggles */
|
||||
@@ -915,7 +938,13 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
return;
|
||||
}
|
||||
updateRunnerDraft((draft) => {
|
||||
const selected = new Set(draft.scenarioIds);
|
||||
const selected = new Set(
|
||||
draft.scenarioIds ??
|
||||
(!state.runnerDraftDirty
|
||||
? state.bootstrap?.runner.plan?.selectedScenarios.map((scenario) => scenario.id)
|
||||
: undefined) ??
|
||||
[],
|
||||
);
|
||||
if (node.checked) {
|
||||
selected.add(scenarioId);
|
||||
} else {
|
||||
@@ -930,6 +959,19 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
});
|
||||
|
||||
/* Config form */
|
||||
root.querySelector<HTMLSelectElement>("#run-profile")?.addEventListener("change", (e) => {
|
||||
const profile = (e.currentTarget as HTMLSelectElement).value;
|
||||
const profileDefaults = state.bootstrap?.runnerCatalog.profiles.find(
|
||||
(entry) => entry.id === profile,
|
||||
);
|
||||
updateRunnerDraft((draft) => ({
|
||||
...draft,
|
||||
profile,
|
||||
channelDriver: profileDefaults?.channelDriver ?? draft.channelDriver,
|
||||
evidenceMode: profileDefaults?.evidenceMode ?? draft.evidenceMode,
|
||||
scenarioIds: null,
|
||||
}));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#provider-mode")?.addEventListener("change", (e) => {
|
||||
const mode =
|
||||
(e.currentTarget as HTMLSelectElement).value === "live-frontier"
|
||||
@@ -941,6 +983,33 @@ export async function createQaLabApp(root: HTMLDivElement) {
|
||||
...defaultModelsForProviderMode(mode, state.bootstrap),
|
||||
}));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#channel-driver")?.addEventListener("change", (e) => {
|
||||
const value = (e.currentTarget as HTMLSelectElement).value;
|
||||
const channelDriver = value === "crabline" || value === "live" ? value : "qa-channel";
|
||||
updateRunnerDraft((draft) => ({ ...draft, channelDriver }));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#execution-channel")?.addEventListener("change", (e) => {
|
||||
const channel = (e.currentTarget as HTMLSelectElement).value.trim() || null;
|
||||
updateRunnerDraft((draft) => ({ ...draft, channel }));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#evidence-mode")?.addEventListener("change", (e) => {
|
||||
const evidenceMode =
|
||||
(e.currentTarget as HTMLSelectElement).value === "slim" ? "slim" : "full";
|
||||
updateRunnerDraft((draft) => ({ ...draft, evidenceMode }));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#runtime-pair")?.addEventListener("change", (e) => {
|
||||
const runtimePair: RunnerSelection["runtimePair"] =
|
||||
(e.currentTarget as HTMLSelectElement).value === "openclaw,codex"
|
||||
? ["openclaw", "codex"]
|
||||
: null;
|
||||
updateRunnerDraft((draft) => ({ ...draft, runtimePair }));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#runtime-pair-lane")?.addEventListener("change", (e) => {
|
||||
const value = (e.currentTarget as HTMLSelectElement).value;
|
||||
const runtimePairLane =
|
||||
value === "core" || value === "extended" || value === "soak" ? value : null;
|
||||
updateRunnerDraft((draft) => ({ ...draft, runtimePairLane }));
|
||||
});
|
||||
root.querySelector<HTMLSelectElement>("#primary-model")?.addEventListener("change", (e) => {
|
||||
const primaryModel = (e.currentTarget as HTMLSelectElement).value;
|
||||
updateRunnerDraft((d) => ({
|
||||
|
||||
@@ -51,7 +51,22 @@ export async function postJson<T>(path: string, body: unknown): Promise<T> {
|
||||
response,
|
||||
path,
|
||||
).catch(() => ({}));
|
||||
throw new Error(payload.error || `${response.status} ${response.statusText}`);
|
||||
throw new QaLabHttpError(
|
||||
payload.error || `${response.status} ${response.statusText}`,
|
||||
response.status,
|
||||
payload,
|
||||
);
|
||||
}
|
||||
return await readJsonResponse<T>(response, path);
|
||||
}
|
||||
|
||||
export class QaLabHttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly payload: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "QaLabHttpError";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -511,6 +511,7 @@ select {
|
||||
|
||||
.scenario-item {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding: 10px 12px;
|
||||
|
||||
@@ -93,7 +93,17 @@ export function renderSidebar(state: UiState): string {
|
||||
const realModels = state.bootstrap?.runnerCatalog.real ?? [];
|
||||
const modelOptions =
|
||||
selection?.providerMode === "live-frontier" && realModels.length > 0 ? realModels : MOCK_MODELS;
|
||||
const selectedIds = new Set(selection?.scenarioIds ?? []);
|
||||
const plan = state.runnerPlanOverride ?? state.bootstrap?.runner.plan ?? null;
|
||||
const resolvedIds = plan?.selectedScenarios.map((scenario) => scenario.id) ?? [];
|
||||
const selectedIds = new Set(
|
||||
selection?.scenarioIds ?? (state.runnerDraftDirty ? [] : resolvedIds),
|
||||
);
|
||||
const profiles = state.bootstrap?.runnerCatalog.profiles ?? [];
|
||||
const channels = state.bootstrap?.runnerCatalog.channels ?? [];
|
||||
const hasRunnableSelection =
|
||||
selection?.scenarioIds === null
|
||||
? Boolean(selection.profile)
|
||||
: Boolean(selection?.scenarioIds.length);
|
||||
|
||||
return `
|
||||
<aside class="sidebar${state.sidebarCollapsed ? " is-collapsed" : ""}">
|
||||
@@ -106,6 +116,17 @@ export function renderSidebar(state: UiState): string {
|
||||
state.sidebarPanel === "config"
|
||||
? `<div class="sidebar-section sidebar-panel-body">
|
||||
<div class="sidebar-section-title"><h3>Configuration</h3></div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Profile</span>
|
||||
<select id="run-profile"${isRunning ? " disabled" : ""}>
|
||||
${profiles
|
||||
.map(
|
||||
(profile) =>
|
||||
`<option value="${esc(profile.id)}"${selection?.profile === profile.id ? " selected" : ""}>${esc(profile.id)}</option>`,
|
||||
)
|
||||
.join("")}
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Provider lane</span>
|
||||
<select id="provider-mode"${isRunning ? " disabled" : ""}>
|
||||
@@ -113,6 +134,52 @@ export function renderSidebar(state: UiState): string {
|
||||
<option value="live-frontier"${selection?.providerMode === "live-frontier" ? " selected" : ""}>Real frontier providers</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Channel driver</span>
|
||||
<select id="channel-driver"${isRunning ? " disabled" : ""}>
|
||||
<option value="qa-channel"${selection?.channelDriver === "qa-channel" ? " selected" : ""}>Synthetic QA channel</option>
|
||||
<option value="crabline"${selection?.channelDriver === "crabline" ? " selected" : ""}>Crabline channel driver</option>
|
||||
<option value="live"${selection?.channelDriver === "live" ? " selected" : ""}>Real channels</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Execution channel</span>
|
||||
<select id="execution-channel"${isRunning ? " disabled" : ""}>
|
||||
<option value=""${selection?.channel ? "" : " selected"}>Catalog/default</option>
|
||||
${channels
|
||||
.map(
|
||||
(channel) =>
|
||||
`<option value="${esc(channel)}"${selection?.channel === channel ? " selected" : ""}>${esc(channel)}</option>`,
|
||||
)
|
||||
.join("")}
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Evidence mode</span>
|
||||
<select id="evidence-mode"${isRunning ? " disabled" : ""}>
|
||||
<option value="full"${selection?.evidenceMode === "full" ? " selected" : ""}>Full</option>
|
||||
<option value="slim"${selection?.evidenceMode === "slim" ? " selected" : ""}>Slim</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Runtime pair</span>
|
||||
<select id="runtime-pair"${isRunning ? " disabled" : ""}>
|
||||
<option value=""${selection?.runtimePair ? "" : " selected"}>Single runtime</option>
|
||||
<option value="openclaw,codex"${selection?.runtimePair ? " selected" : ""}>OpenClaw × Codex</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-field">
|
||||
<span class="config-label">Runtime-pair lane</span>
|
||||
<select id="runtime-pair-lane"${isRunning ? " disabled" : ""}>
|
||||
<option value=""${selection?.runtimePairLane ? "" : " selected"}>Profile/default</option>
|
||||
${(["core", "extended", "soak"] as const)
|
||||
.map(
|
||||
(lane) =>
|
||||
`<option value="${lane}"${selection?.runtimePairLane === lane ? " selected" : ""}>${lane}</option>`,
|
||||
)
|
||||
.join("")}
|
||||
</select>
|
||||
</div>
|
||||
${renderModelSelect({
|
||||
id: "primary-model",
|
||||
label: "Primary model",
|
||||
@@ -143,10 +210,10 @@ export function renderSidebar(state: UiState): string {
|
||||
? `<div class="sidebar-panel-body">${run || runner ? renderRunStatus(state) : '<div class="sidebar-section"><div class="text-dimmed text-sm">No run data yet.</div></div>'}</div>`
|
||||
: `<div class="sidebar-section sidebar-scenarios sidebar-panel-body">
|
||||
<div class="sidebar-section-title">
|
||||
<h3>Scenarios (${selectedIds.size}/${scenarios.length})</h3>
|
||||
<h3>Scenarios (${selection?.scenarioIds === null ? "profile" : selectedIds.size}/${scenarios.length})</h3>
|
||||
<div class="btn-group">
|
||||
<button class="btn-sm btn-ghost" data-action="select-all-scenarios"${isRunning ? " disabled" : ""}>All</button>
|
||||
<button class="btn-sm btn-ghost" data-action="clear-scenarios"${isRunning ? " disabled" : ""}>None</button>
|
||||
<button class="btn-sm btn-ghost" data-action="clear-scenarios"${isRunning ? " disabled" : ""}>Profile</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="scenario-scroll">
|
||||
@@ -160,7 +227,7 @@ export function renderSidebar(state: UiState): string {
|
||||
<span class="${statusDotClass(status)}"></span>
|
||||
<div class="scenario-item-info">
|
||||
<span class="scenario-item-title">${esc(s.title)}</span>
|
||||
<span class="scenario-item-meta">${esc(s.surface)} · ${esc(s.id)}</span>
|
||||
<span class="scenario-item-meta">${esc(s.surface)} · ${esc(s.execution?.kind ?? "flow")} · ${esc(s.id)}</span>
|
||||
</div>
|
||||
</label>`;
|
||||
})
|
||||
@@ -171,8 +238,8 @@ export function renderSidebar(state: UiState): string {
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="sidebar-actions">
|
||||
<button class="btn-primary" data-action="run-suite"${isRunning || !selectedIds.size || state.busy ? " disabled" : ""}>
|
||||
Run ${selectedIds.size} scenario${selectedIds.size === 1 ? "" : "s"}
|
||||
<button class="btn-primary" data-action="run-suite"${isRunning || !hasRunnableSelection || state.busy ? " disabled" : ""}>
|
||||
${selection?.scenarioIds === null ? `Resolve & run ${esc(selection.profile)}` : `Run ${selectedIds.size} scenario${selectedIds.size === 1 ? "" : "s"}`}
|
||||
</button>
|
||||
<div class="btn-row">
|
||||
<button data-action="self-check"${isRunning || state.busy ? " disabled" : ""}>Self-check</button>
|
||||
@@ -185,6 +252,7 @@ export function renderSidebar(state: UiState): string {
|
||||
function renderRunStatus(state: UiState): string {
|
||||
const run = state.scenarioRun;
|
||||
const runner = state.bootstrap?.runner ?? null;
|
||||
const plan = state.runnerPlanOverride ?? (state.runnerDraftDirty ? null : (runner?.plan ?? null));
|
||||
if (!run && !runner) {
|
||||
return "";
|
||||
}
|
||||
@@ -206,6 +274,9 @@ function renderRunStatus(state: UiState): string {
|
||||
: ""
|
||||
}
|
||||
<div class="run-meta">
|
||||
${plan ? `<strong>Resolved plan:</strong> ${plan.selectedScenarios.length} selected · ${esc(plan.executionKinds.join(", ") || "none")}` : ""}
|
||||
${plan?.exclusions.length ? `<br>${plan.exclusions.length} excluded: ${esc(plan.exclusions.map((item) => `${item.scenarioId} (${item.reasons.join(", ")})`).join("; "))}` : ""}
|
||||
${plan?.errors.length ? `<br><span style="color:var(--danger)">${esc(plan.errors.join(" "))}</span>` : ""}
|
||||
${runner?.startedAt ? `Started ${esc(formatIso(runner.startedAt))}` : ""}
|
||||
${runner?.finishedAt ? `<br>Finished ${esc(formatIso(runner.finishedAt))}` : ""}
|
||||
${runner?.error ? `<br><span style="color:var(--danger)">${esc(runner.error)}</span>` : ""}
|
||||
|
||||
@@ -68,6 +68,7 @@ function evidenceState(overrides: Partial<UiState> = {}): UiState {
|
||||
latestReport: null,
|
||||
runnerDraft: null,
|
||||
runnerDraftDirty: false,
|
||||
runnerPlanOverride: null,
|
||||
scenarioRun: null,
|
||||
selectedCaptureEventKey: null,
|
||||
selectedCaptureSessionIds: [],
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
import type {
|
||||
QaLabExecutionKind,
|
||||
QaLabResolvedRunPlan,
|
||||
QaLabRunnerSnapshot,
|
||||
QaLabRunSelection,
|
||||
} from "../../runner-contract.js";
|
||||
import type {
|
||||
QaEvidenceArtifactView,
|
||||
QaEvidenceGalleryEntryView,
|
||||
@@ -82,6 +88,11 @@ export type SeedScenario = {
|
||||
successCriteria: string[];
|
||||
docsRefs?: string[];
|
||||
codeRefs?: string[];
|
||||
execution?: {
|
||||
kind?: QaLabExecutionKind;
|
||||
channel?: string;
|
||||
};
|
||||
runtimePairLane?: "core" | "extended" | "soak";
|
||||
};
|
||||
|
||||
export type Bootstrap = {
|
||||
@@ -101,6 +112,13 @@ export type Bootstrap = {
|
||||
runnerCatalog: {
|
||||
status: "loading" | "ready" | "failed";
|
||||
real: RunnerModelOption[];
|
||||
channels: string[];
|
||||
profiles: Array<{
|
||||
id: string;
|
||||
evidenceMode: "full" | "slim";
|
||||
channelDriver: "qa-channel" | "crabline" | "live";
|
||||
categoryIds: string[];
|
||||
}>;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -136,28 +154,9 @@ type ScenarioRun = {
|
||||
};
|
||||
};
|
||||
|
||||
export type RunnerSelection = {
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode: boolean;
|
||||
scenarioIds: string[];
|
||||
};
|
||||
|
||||
type RunnerSnapshot = {
|
||||
status: "idle" | "running" | "completed" | "failed";
|
||||
selection: RunnerSelection;
|
||||
startedAt?: string;
|
||||
finishedAt?: string;
|
||||
artifacts: null | {
|
||||
evidencePath: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
watchUrl: string;
|
||||
};
|
||||
error: string | null;
|
||||
};
|
||||
export type RunnerSelection = QaLabRunSelection;
|
||||
export type RunnerResolvedPlan = QaLabResolvedRunPlan;
|
||||
type RunnerSnapshot = QaLabRunnerSnapshot;
|
||||
|
||||
export type RunnerModelOption = {
|
||||
key: string;
|
||||
@@ -370,6 +369,7 @@ export type UiState = {
|
||||
activeTab: TabId;
|
||||
runnerDraft: RunnerSelection | null;
|
||||
runnerDraftDirty: boolean;
|
||||
runnerPlanOverride: RunnerResolvedPlan | null;
|
||||
composer: {
|
||||
conversationKind: "direct" | "channel";
|
||||
conversationId: string;
|
||||
|
||||
Reference in New Issue
Block a user