From e8db9c3bc0dffa48c2d1577e7725dae75eee2d06 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 14 Jun 2026 18:08:42 -0700 Subject: [PATCH] test(qa): add qa run --profile and unified output summary/evidence (#91587) * test(qa): add mapped qa run profiles * test(qa): document mapped profile runner * test(qa): validate run profiles from mapping * test(qa): preserve root profile parsing * test(qa): simplify taxonomy profile dispatch * test(qa): align tool coverage CLI expectation * test(qa): fix profile dispatch fixture type * test(qa): share profile runner option types * test(qa): split shared cli runner options * test(qa): unify profile suite artifacts * fix(qa): filter profile scenarios by provider lane * test(qa): drop native scenario subreports * fix(qa): keep native log refs repo-relative * fix(cli): preserve qa run root profile parsing * fix(qa): avoid qa profile flag collision * fix(qa): reject profile flags without qa profile --- docs/concepts/qa-e2e-automation.md | 28 +- docs/help/testing.md | 3 + extensions/qa-lab/src/cli.runtime.test.ts | 87 ++++- extensions/qa-lab/src/cli.runtime.ts | 201 +++++++++-- extensions/qa-lab/src/cli.test.ts | 121 +++++++ extensions/qa-lab/src/cli.ts | 248 +++++++++----- .../qa-lab/src/suite-launch.runtime.test.ts | 153 +++++++-- extensions/qa-lab/src/suite-launch.runtime.ts | 316 ++++++++++++++++-- extensions/qa-lab/src/suite-planning.ts | 5 +- .../src/test-file-scenario-runner.test.ts | 11 - .../qa-lab/src/test-file-scenario-runner.ts | 84 +---- src/cli/profile.test.ts | 122 +++++++ src/cli/profile.ts | 20 +- 13 files changed, 1130 insertions(+), 269 deletions(-) diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 134ec5c37dee..5c2841c322d1 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -31,7 +31,7 @@ script aliases; both forms are supported. | Command | Purpose | | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `qa run` | Bundled QA self-check; writes a Markdown report. | +| `qa run` | Bundled QA self-check without `--qa-profile`; taxonomy-backed maturity profile runner with `--qa-profile smoke-ci` or `--qa-profile release`. | | `qa suite` | Run repo-backed scenarios against the QA gateway lane. Aliases: `pnpm openclaw qa suite --runner multipass` for a disposable Linux VM. | | `qa coverage` | Print the YAML scenario-coverage inventory (`--json` for machine output). | | `qa parity-report` | Compare two `qa-suite-summary.json` files and write the agentic parity report, or use `--runtime-axis --token-efficiency` to write Codex-vs-OpenClaw runtime parity and token-efficiency reports from one runtime-pair summary. | @@ -51,6 +51,26 @@ script aliases; both forms are supported. | `qa whatsapp` | Live transport lane against real WhatsApp Web accounts. | | `qa mantis` | Before and after verification runner for live transport bugs, with Discord status-reactions evidence, Crabbox desktop/browser smoke, and Slack-in-VNC smoke. See [Mantis](/concepts/mantis) and [Mantis Slack Desktop Runbook](/concepts/mantis-slack-desktop-runbook). | +Profile-backed `qa run` reads membership from `taxonomy.yaml`, then dispatches +the resolved scenarios through `qa suite`. `--surface` and +`--category` filter the selected profile instead of defining separate lanes: + +```bash +pnpm openclaw qa run \ + --qa-profile smoke-ci \ + --category agent-runtime-and-provider-execution.agent-turn-execution \ + --provider-mode mock-openai \ + --output-dir .artifacts/qa-e2e/smoke-ci-profile-dispatch +``` + +Use `smoke-ci` for deterministic no-live-service proof and `release` for the +Stable/LTS proof lane. When a command also needs an OpenClaw root profile, put +the root profile before the QA command: + +```bash +pnpm openclaw --profile work qa run --qa-profile smoke-ci +``` + ## Operator flow The current QA operator flow is a two-pane QA site: @@ -913,7 +933,11 @@ The report should answer: For the inventory of available scenarios - useful when sizing follow-up work or wiring a new transport - run `pnpm openclaw qa coverage` (add `--json` for machine-readable output). When choosing focused proof for a touched behavior or file path, run `pnpm openclaw qa coverage --match `. The match report searches scenario metadata, docs refs, code refs, coverage IDs, plugins, and provider requirements, then prints matching `qa suite --scenario ...` targets. -Every `qa suite` scenario execution writes a `qa-evidence.json` artifact. Flow scenarios also write `qa-suite-summary.json` for existing suite/report tooling; scenarios that declare `execution.kind: vitest` or `execution.kind: playwright` run the matching test path and write `qa-vitest-report.md` or `qa-playwright-report.md` plus per-scenario logs. +Every `qa suite` run writes top-level `qa-evidence.json`, +`qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected +scenario set. Scenarios that declare `execution.kind: vitest` or +`execution.kind: playwright` run the matching test path and also write +per-scenario logs. Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test. For character and style checks, run the same scenario across multiple live model diff --git a/docs/help/testing.md b/docs/help/testing.md index 82c0abb59119..d577741c1417 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -145,6 +145,9 @@ inside every shard. - `pnpm openclaw qa suite` - Runs repo-backed QA scenarios directly on the host. + - Writes top-level `qa-evidence.json`, `qa-suite-summary.json`, and + `qa-suite-report.md` artifacts for the selected scenario set, including + mixed flow, Vitest, and Playwright scenario selections. - Runs multiple selected scenarios in parallel by default with isolated gateway workers. `qa-channel` defaults to concurrency 4 (bounded by the selected scenario count). Use `--concurrency ` to tune the worker diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index fd3cec129b01..79037c96e404 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -83,6 +83,7 @@ import { runQaJsonlReplayCommand, runQaManualLaneCommand, runQaParityReportCommand, + runQaProfileCommand, runQaSuiteCommand, } from "./cli.runtime.js"; import { QaSuiteInfraError } from "./errors.js"; @@ -139,21 +140,22 @@ function flowSuiteRuntimeResult(params: { }; } -function testFileSuiteRuntimeResult(params: { +function unifiedSuiteRuntimeResult(params: { evidencePath: string; - executionKind?: "vitest" | "playwright"; outputDir: string; reportPath: string; - results?: unknown[]; + summaryPath: string; + scenarios?: unknown[]; }) { return { - executionKind: params.executionKind ?? "playwright", + executionKind: "suite", result: { outputDir: params.outputDir, - executionKind: params.executionKind ?? "playwright", reportPath: params.reportPath, evidencePath: params.evidencePath, - results: params.results ?? [{ status: "pass" }], + summaryPath: params.summaryPath, + report: "# QA Suite Report\n", + scenarios: params.scenarios ?? [], }, }; } @@ -301,9 +303,10 @@ describe("qa cli runtime", () => { const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json"); await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8"); runQaSuite.mockResolvedValueOnce( - testFileSuiteRuntimeResult({ + unifiedSuiteRuntimeResult({ outputDir: suiteArtifactsDir, reportPath: suiteReportPath, + summaryPath: suiteSummaryPath, evidencePath, }), ); @@ -325,6 +328,7 @@ describe("qa cli runtime", () => { scenarioIds: ["control-ui-chat-flow-playwright"], }); expectWriteContains(stdoutWrite, `QA suite evidence: ${evidencePath}`); + expectWriteContains(stdoutWrite, `QA suite summary: ${suiteSummaryPath}`); }); it("rejects host-only resource options for Playwright scenarios", async () => { @@ -339,6 +343,75 @@ describe("qa cli runtime", () => { expect(runQaSuite).not.toHaveBeenCalled(); }); + it("dispatches a taxonomy-backed profile category through the suite runner", async () => { + const previousProfile = process.env.OPENCLAW_QA_PROFILE; + process.env.OPENCLAW_QA_PROFILE = "release"; + try { + runQaSuite.mockImplementationOnce(async () => { + expect(process.env.OPENCLAW_QA_PROFILE).toBe("smoke-ci"); + return flowSuiteRuntimeResult({ + reportPath: suiteReportPath, + summaryPath: suiteSummaryPath, + }); + }); + + await runQaProfileCommand({ + repoRoot: "/tmp/openclaw-repo", + outputDir: ".artifacts/qa-e2e/smoke-ci", + profile: "smoke-ci", + surface: "agent-runtime-and-provider-execution", + category: "agent-runtime-and-provider-execution.agent-turn-execution", + transportId: "qa-channel", + fastMode: true, + concurrency: 2, + allowFailures: true, + }); + + const suiteArgs = mockFirstObjectArg(runQaSuite); + expectFields(suiteArgs, { + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa-e2e/smoke-ci"), + transportId: "qa-channel", + providerMode: "mock-openai", + fastMode: true, + concurrency: 2, + }); + expect(suiteArgs.scenarioIds).toEqual(expect.arrayContaining(["dm-chat-baseline"])); + expect(suiteArgs.scenarioIds).not.toContain("thinking-slash-model-remap"); + expect(process.env.OPENCLAW_QA_PROFILE).toBe("release"); + expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:"); + } finally { + if (previousProfile === undefined) { + delete process.env.OPENCLAW_QA_PROFILE; + } else { + process.env.OPENCLAW_QA_PROFILE = previousProfile; + } + } + }); + + it("rejects qa profile runs that do not match taxonomy categories", async () => { + await expect( + runQaProfileCommand({ + repoRoot: "/tmp/openclaw-repo", + profile: "smoke-ci", + surface: "unknown-surface", + }), + ).rejects.toThrow( + "qa run did not find taxonomy categories for --qa-profile smoke-ci --surface unknown-surface.", + ); + expect(runQaSuite).not.toHaveBeenCalled(); + }); + + it("rejects qa profile runs whose profile is not declared in taxonomy.yaml", async () => { + await expect( + runQaProfileCommand({ + repoRoot: "/tmp/openclaw-repo", + profile: "nightly", + }), + ).rejects.toThrow('--qa-profile must be one of smoke-ci, release, got "nightly".'); + expect(runQaSuite).not.toHaveBeenCalled(); + }); + it("resolves suite repo-root-relative paths before dispatching", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 18741b4311f6..0902b88075e2 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -68,7 +68,12 @@ import { type QaRuntimeParityTier, } from "./scenario-catalog.js"; import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js"; +import { + readQaScorecardTaxonomyReport, + type QaScorecardCategoryMappingReport, +} from "./scorecard-taxonomy.js"; import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js"; +import { scenarioMatchesQaProviderLane } from "./suite-planning.js"; import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js"; import { buildTokenEfficiencyReport, @@ -95,6 +100,49 @@ type InterruptibleServer = { stop(): Promise; }; +export type QaLabSelfCheckCommandOptions = { + repoRoot?: string; + output?: string; +}; + +type QaScenarioProviderCommandOptions = { + transportId?: string; + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; +}; + +type QaScenarioRunCommandOptions = QaScenarioProviderCommandOptions & { + repoRoot?: string; + outputDir?: string; + concurrency?: number; + allowFailures?: boolean; +}; + +export type QaProfileCommandOptions = QaScenarioRunCommandOptions & { + profile: string; + surface?: string; + category?: string; +}; + +export type QaSuiteCommandOptions = QaScenarioRunCommandOptions & { + runner?: string; + thinking?: string; + cliAuthMode?: string; + parityPack?: string; + pack?: string; + scenarioIds?: string[]; + enabledPluginIds?: string[]; + image?: string; + cpus?: number; + memory?: string; + disk?: string; + preflight?: boolean; + runtimePair?: string; + runtimeParityTier?: string[]; +}; + function resolveQaManualLaneModels(opts: { providerMode: QaProviderMode; primaryModel?: string; @@ -557,7 +605,7 @@ function printQaCredentialDoctorTable( } } -export async function runQaLabSelfCheckCommand(opts: { repoRoot?: string; output?: string }) { +export async function runQaLabSelfCheckCommand(opts: QaLabSelfCheckCommandOptions) { const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const server = await startQaLabServer({ repoRoot, @@ -571,31 +619,126 @@ export async function runQaLabSelfCheckCommand(opts: { repoRoot?: string; output } } -export async function runQaSuiteCommand(opts: { - repoRoot?: string; - outputDir?: string; - transportId?: string; - runner?: string; - providerMode?: QaProviderModeInput; - primaryModel?: string; - alternateModel?: string; - fastMode?: boolean; - thinking?: string; - cliAuthMode?: string; - parityPack?: string; - pack?: string; - scenarioIds?: string[]; - concurrency?: number; - allowFailures?: boolean; - enabledPluginIds?: string[]; - image?: string; - cpus?: number; - memory?: string; - disk?: string; - preflight?: boolean; - runtimePair?: string; - runtimeParityTier?: string[]; -}) { +export async function runQaProfileCommand(opts: QaProfileCommandOptions) { + const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); + const scenarioPack = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(scenarioPack.scenarios); + const profile = normalizeQaRunProfile( + opts.profile, + scorecardReport.profiles.map((entry) => entry.id), + ); + const categories = scorecardReport.categories.filter((category) => + qaScorecardCategoryMatchesRunProfile(category, { + profile, + surface: opts.surface, + category: opts.category, + }), + ); + if (categories.length === 0) { + throw new Error(formatQaRunProfileNoMatchMessage(opts)); + } + + const scenarioBySourcePath = new Map( + scenarioPack.scenarios.map((scenario) => [scenario.sourcePath, scenario] as const), + ); + const taxonomyScenarios = uniqueStrings(categories.flatMap((category) => category.scenarioRefs)) + .map((scenarioRef) => scenarioBySourcePath.get(scenarioRef)) + .filter((scenario): scenario is NonNullable => scenario !== undefined); + const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile); + const normalizedProviderMode = normalizeQaProviderMode(providerMode); + const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode); + const scenarios = taxonomyScenarios.filter((scenario) => + scenarioMatchesQaProviderLane({ + scenario, + providerMode: normalizedProviderMode, + primaryModel, + }), + ); + if (scenarios.length === 0) { + throw new Error( + `qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`, + ); + } + + process.stdout.write( + `QA run profile: ${profile}; categories: ${categories.length}; scenarios: ${scenarios.length}\n`, + ); + await withTemporaryQaProfileEnv(profile, async () => { + await runQaSuiteCommand({ + repoRoot, + outputDir: opts.outputDir, + transportId: opts.transportId, + providerMode, + primaryModel: opts.primaryModel, + alternateModel: opts.alternateModel, + fastMode: opts.fastMode, + scenarioIds: scenarios.map((scenario) => scenario.id), + concurrency: opts.concurrency, + allowFailures: opts.allowFailures, + }); + }); +} + +function normalizeQaRunProfile(value: string, profileIds: readonly string[]) { + if (profileIds.length === 0) { + throw new Error("taxonomy.yaml does not define QA run profiles."); + } + const normalized = value.trim(); + if (profileIds.includes(normalized)) { + return normalized; + } + throw new Error(`--qa-profile must be one of ${profileIds.join(", ")}, got "${value}".`); +} + +function defaultQaRunProfileProviderMode(profile: string): QaProviderModeInput { + return profile === "smoke-ci" ? "mock-openai" : DEFAULT_QA_LIVE_PROVIDER_MODE; +} + +function qaScorecardCategoryMatchesRunProfile( + category: QaScorecardCategoryMappingReport, + opts: { profile: string; surface?: string; category?: string }, +): boolean { + if (!category.profiles.includes(opts.profile)) { + return false; + } + if (opts.surface?.trim()) { + const surface = opts.surface.trim(); + if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) { + return false; + } + } + if (opts.category?.trim() && category.id !== opts.category.trim()) { + return false; + } + return true; +} + +function formatQaRunProfileNoMatchMessage( + opts: Pick, +) { + const filters = [ + `--qa-profile ${opts.profile}`, + opts.surface?.trim() ? `--surface ${opts.surface.trim()}` : null, + opts.category?.trim() ? `--category ${opts.category.trim()}` : null, + ].filter((filter): filter is string => filter !== null); + return `qa run did not find taxonomy categories for ${filters.join(" ")}.`; +} + +async function withTemporaryQaProfileEnv(profile: string, run: () => Promise): Promise { + const previousProfile = process.env.OPENCLAW_QA_PROFILE; + process.env.OPENCLAW_QA_PROFILE = profile; + try { + return await run(); + } finally { + if (previousProfile === undefined) { + delete process.env.OPENCLAW_QA_PROFILE; + } else { + process.env.OPENCLAW_QA_PROFILE = previousProfile; + } + } +} + +export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const transportId = normalizeQaTransportId(opts.transportId); const runner = (opts.runner ?? "host").trim().toLowerCase(); @@ -705,12 +848,12 @@ export async function runQaSuiteCommand(opts: { }), ); switch (runtimeResult.executionKind) { - case "vitest": - case "playwright": { + case "suite": { const result = runtimeResult.result; process.stdout.write(`QA suite report: ${result.reportPath}\n`); process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`); - if (!allowFailures && result.results.some((scenario) => scenario.status !== "pass")) { + process.stdout.write(`QA suite summary: ${result.summaryPath}\n`); + if (!allowFailures && result.scenarios.some((scenario) => scenario.status !== "pass")) { process.exitCode = 1; } return; diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 33987c3b0e1c..5dd1d87d15c0 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -47,6 +47,8 @@ const { runQaCredentialsRemoveCommand, runQaCoverageReportCommand, runQaJsonlReplayCommand, + runQaLabSelfCheckCommand, + runQaProfileCommand, runQaProviderServerCommand, runQaSuiteCommand, runQaTelegramCommand, @@ -61,6 +63,8 @@ const { runQaCredentialsRemoveCommand: vi.fn(), runQaCoverageReportCommand: vi.fn(), runQaJsonlReplayCommand: vi.fn(), + runQaLabSelfCheckCommand: vi.fn(), + runQaProfileCommand: vi.fn(), runQaProviderServerCommand: vi.fn(), runQaSuiteCommand: vi.fn(), runQaTelegramCommand: vi.fn(), @@ -117,6 +121,8 @@ vi.mock("./cli.runtime.js", () => ({ runQaCredentialsRemoveCommand, runQaCoverageReportCommand, runQaJsonlReplayCommand, + runQaLabSelfCheckCommand, + runQaProfileCommand, runQaProviderServerCommand, runQaSuiteCommand, })); @@ -133,6 +139,8 @@ describe("qa cli registration", () => { runQaCredentialsRemoveCommand.mockReset(); runQaCoverageReportCommand.mockReset(); runQaJsonlReplayCommand.mockReset(); + runQaLabSelfCheckCommand.mockReset(); + runQaProfileCommand.mockReset(); runQaProviderServerCommand.mockReset(); runQaSuiteCommand.mockReset(); runQaTelegramCommand.mockReset(); @@ -174,6 +182,119 @@ describe("qa cli registration", () => { expect(ui.options.map((option) => option.long)).not.toContain("--control-ui-token"); }); + it("keeps qa run without a profile on the self-check command", async () => { + await program.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--repo-root", + "/tmp/openclaw-repo", + "--output", + ".artifacts/qa-self-check.md", + ]); + + expect(runQaLabSelfCheckCommand).toHaveBeenCalledWith({ + repoRoot: "/tmp/openclaw-repo", + output: ".artifacts/qa-self-check.md", + }); + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + + it("routes qa run qa-profile flags into the taxonomy-backed profile command", async () => { + await program.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--repo-root", + "/tmp/openclaw-repo", + "--output-dir", + ".artifacts/qa-e2e/smoke-ci", + "--qa-profile", + "smoke-ci", + "--surface", + "agent-runtime-and-provider-execution", + "--category", + "agent-runtime-and-provider-execution.agent-turn-execution", + "--transport", + "qa-channel", + "--provider-mode", + "mock-openai", + "--model", + "openai/gpt-5.5", + "--alt-model", + "anthropic/claude-sonnet-4-6", + "--concurrency", + "2", + "--allow-failures", + "--fast", + ]); + + expect(runQaProfileCommand).toHaveBeenCalledWith({ + repoRoot: "/tmp/openclaw-repo", + outputDir: ".artifacts/qa-e2e/smoke-ci", + profile: "smoke-ci", + surface: "agent-runtime-and-provider-execution", + category: "agent-runtime-and-provider-execution.agent-turn-execution", + transportId: "qa-channel", + providerMode: "mock-openai", + primaryModel: "openai/gpt-5.5", + alternateModel: "anthropic/claude-sonnet-4-6", + concurrency: 2, + allowFailures: true, + fastMode: true, + }); + expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled(); + }); + + it.each([ + ["--output-dir", [".artifacts/qa-e2e/smoke-ci"]], + ["--surface", ["agent-runtime-and-provider-execution"]], + ["--category", ["agent-runtime-and-provider-execution.agent-turn-execution"]], + ["--transport", ["qa-channel"]], + ["--provider-mode", ["mock-openai"]], + ["--model", ["openai/gpt-5.5"]], + ["--alt-model", ["anthropic/claude-sonnet-4-6"]], + ["--concurrency", ["2"]], + ["--allow-failures", []], + ["--fast", []], + ])("rejects qa run profile-only flag %s without --qa-profile", async (flag, values) => { + await expect( + program.parseAsync(["node", "openclaw", "qa", "run", flag, ...values]), + ).rejects.toThrow(`qa run ${flag} requires --qa-profile`); + + expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled(); + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + + it("rejects an empty qa run --qa-profile instead of falling back to self-check", async () => { + await expect( + program.parseAsync(["node", "openclaw", "qa", "run", "--qa-profile", ""]), + ).rejects.toThrow("--qa-profile must not be empty."); + + expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled(); + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + + it("rejects self-check output flags in qa run profile mode", async () => { + await expect( + program.parseAsync([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "smoke-ci", + "--output", + ".artifacts/qa-self-check.md", + ]), + ).rejects.toThrow("qa run --output is only valid for the self-check mode"); + + expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled(); + expect(runQaProfileCommand).not.toHaveBeenCalled(); + }); + it("routes mantis discord-smoke flags into the mantis runtime command", async () => { await program.parseAsync([ "node", diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index 16063567141d..1e1ed1289eb6 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -2,6 +2,11 @@ import type { Command } from "commander"; import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; import { collectString } from "./cli-options.js"; +import type { + QaLabSelfCheckCommandOptions, + QaProfileCommandOptions, + QaSuiteCommandOptions, +} from "./cli.runtime.js"; import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js"; import { registerMantisCli } from "./mantis/cli.js"; import { @@ -18,6 +23,57 @@ import { hasQaScenarioPack } from "./scenario-catalog.js"; type QaLabCliRuntime = typeof import("./cli.runtime.js"); +type QaScenarioRunCliOptions = { + repoRoot?: QaSuiteCommandOptions["repoRoot"]; + outputDir?: QaSuiteCommandOptions["outputDir"]; + transport?: QaSuiteCommandOptions["transportId"]; + providerMode?: QaSuiteCommandOptions["providerMode"]; + model?: QaSuiteCommandOptions["primaryModel"]; + altModel?: QaSuiteCommandOptions["alternateModel"]; + concurrency?: QaSuiteCommandOptions["concurrency"]; + allowFailures?: QaSuiteCommandOptions["allowFailures"]; + fast?: QaSuiteCommandOptions["fastMode"]; +}; + +type QaRunCliOptions = QaLabSelfCheckCommandOptions & + QaScenarioRunCliOptions & { + qaProfile?: QaProfileCommandOptions["profile"]; + surface?: QaProfileCommandOptions["surface"]; + category?: QaProfileCommandOptions["category"]; + }; + +const QA_RUN_PROFILE_ONLY_OPTIONS = [ + { optionName: "outputDir", flag: "--output-dir" }, + { optionName: "surface", flag: "--surface" }, + { optionName: "category", flag: "--category" }, + { optionName: "transport", flag: "--transport" }, + { optionName: "providerMode", flag: "--provider-mode" }, + { optionName: "model", flag: "--model" }, + { optionName: "altModel", flag: "--alt-model" }, + { optionName: "concurrency", flag: "--concurrency" }, + { optionName: "allowFailures", flag: "--allow-failures" }, + { optionName: "fast", flag: "--fast" }, +] as const; + +const QA_RUN_SELF_CHECK_ONLY_OPTIONS = [{ optionName: "output", flag: "--output" }] as const; + +type QaSuiteCliOptions = QaScenarioRunCliOptions & { + runner?: QaSuiteCommandOptions["runner"]; + thinking?: QaSuiteCommandOptions["thinking"]; + cliAuthMode?: QaSuiteCommandOptions["cliAuthMode"]; + parityPack?: QaSuiteCommandOptions["parityPack"]; + pack?: QaSuiteCommandOptions["pack"]; + scenario?: QaSuiteCommandOptions["scenarioIds"]; + enablePlugin?: QaSuiteCommandOptions["enabledPluginIds"]; + image?: QaSuiteCommandOptions["image"]; + cpus?: QaSuiteCommandOptions["cpus"]; + memory?: QaSuiteCommandOptions["memory"]; + disk?: QaSuiteCommandOptions["disk"]; + preflight?: QaSuiteCommandOptions["preflight"]; + runtimePair?: QaSuiteCommandOptions["runtimePair"]; + runtimeParityTier?: QaSuiteCommandOptions["runtimeParityTier"]; +}; + let qaLabCliRuntimePromise: Promise | null = null; async function loadQaLabCliRuntime(): Promise { @@ -41,36 +97,54 @@ function parseQaCliPositiveIntegerOption(value: string, flag: string): number { return parsed; } -async function runQaSelfCheck(opts: { repoRoot?: string; output?: string }) { +function collectCliSuppliedQaRunFlags( + command: Command, + options: readonly { optionName: string; flag: string }[], +): string[] { + return options + .filter((option) => command.getOptionValueSource(option.optionName) === "cli") + .map((option) => option.flag); +} + +function formatFlagList(flags: readonly string[]): string { + return flags.length === 1 ? flags[0] : flags.join(", "); +} + +function validateQaRunMode(opts: QaRunCliOptions, command: Command) { + const hasQaProfile = Boolean(opts.qaProfile?.trim()); + if (command.getOptionValueSource("qaProfile") === "cli" && !hasQaProfile) { + throw new Error("--qa-profile must not be empty."); + } + + if (hasQaProfile) { + const selfCheckFlags = collectCliSuppliedQaRunFlags(command, QA_RUN_SELF_CHECK_ONLY_OPTIONS); + if (selfCheckFlags.length > 0) { + throw new Error( + `qa run ${formatFlagList(selfCheckFlags)} is only valid for the self-check mode without --qa-profile.`, + ); + } + return; + } + + const profileFlags = collectCliSuppliedQaRunFlags(command, QA_RUN_PROFILE_ONLY_OPTIONS); + if (profileFlags.length > 0) { + throw new Error( + `qa run ${formatFlagList(profileFlags)} requires --qa-profile; without --qa-profile, qa run only executes the self-check.`, + ); + } +} + +async function runQaSelfCheck(opts: QaLabSelfCheckCommandOptions) { const runtime = await loadQaLabCliRuntime(); await runtime.runQaLabSelfCheckCommand(opts); } -async function runQaSuiteCliCommand(opts: { - repoRoot?: string; - outputDir?: string; - transportId?: string; - providerMode?: QaProviderModeInput; - primaryModel?: string; - alternateModel?: string; - fastMode?: boolean; - thinking?: string; - allowFailures?: boolean; - enabledPluginIds?: string[]; - cliAuthMode?: string; - parityPack?: string; - pack?: string; - scenarioIds?: string[]; - concurrency?: number; - runner?: string; - image?: string; - cpus?: number; - memory?: string; - disk?: string; - preflight?: boolean; - runtimePair?: string; - runtimeParityTier?: string[]; -}) { +async function runQaProfile(opts: QaProfileCommandOptions) { + const runtime = await loadQaLabCliRuntime(); + await runtime.runQaProfileCommand(opts); +} + +async function runQaSuiteCliCommand(opts: QaSuiteCommandOptions) { const runtime = await loadQaLabCliRuntime(); await runtime.runQaSuiteCommand(opts); } @@ -290,8 +364,46 @@ export function registerQaLabCli(program: Command) { .description("Run the bundled QA self-check and write a Markdown report") .option("--repo-root ", "Repository root to target when running from a neutral cwd") .option("--output ", "Report output path") - .action(async (opts: { repoRoot?: string; output?: string }) => { - await runQaSelfCheck(opts); + .option("--output-dir ", "Profile run artifact directory") + .option("--qa-profile ", "Run the QA profile from taxonomy.yaml") + .option("--surface ", "Limit --qa-profile to a taxonomy surface id") + .option("--category ", "Limit --qa-profile to a taxonomy category id") + .option("--transport ", "QA transport id", "qa-channel") + .option("--provider-mode ", formatQaProviderModeHelp()) + .option("--model ", "Primary provider/model ref") + .option("--alt-model ", "Alternate provider/model ref") + .option("--concurrency ", "Scenario worker concurrency", (value: string) => + parseQaCliPositiveIntegerOption(value, "--concurrency"), + ) + .option( + "--allow-failures", + "Write artifacts without setting a failing exit code when scenarios fail", + false, + ) + .option("--fast", "Enable provider fast mode where supported", false) + .action(async (opts: QaRunCliOptions, command: Command) => { + validateQaRunMode(opts, command); + if (opts.qaProfile?.trim()) { + await runQaProfile({ + repoRoot: opts.repoRoot, + outputDir: opts.outputDir, + profile: opts.qaProfile, + surface: opts.surface, + category: opts.category, + transportId: opts.transport, + providerMode: opts.providerMode, + primaryModel: opts.model, + alternateModel: opts.altModel, + concurrency: opts.concurrency, + allowFailures: opts.allowFailures, + fastMode: opts.fast, + }); + return; + } + await runQaSelfCheck({ + repoRoot: opts.repoRoot, + output: opts.output, + }); }); qa.command("suite") @@ -346,59 +458,33 @@ export function registerQaLabCli(program: Command) { collectString, [], ) - .action( - async (opts: { - repoRoot?: string; - outputDir?: string; - transport?: string; - runner?: string; - providerMode?: QaProviderModeInput; - model?: string; - altModel?: string; - cliAuthMode?: string; - parityPack?: string; - pack?: string; - scenario?: string[]; - enablePlugin?: string[]; - concurrency?: number; - allowFailures?: boolean; - fast?: boolean; - thinking?: string; - image?: string; - cpus?: number; - memory?: string; - disk?: string; - preflight?: boolean; - runtimePair?: string; - runtimeParityTier?: string[]; - }) => { - await runQaSuiteCliCommand({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - transportId: opts.transport, - runner: opts.runner, - providerMode: opts.providerMode, - primaryModel: opts.model, - alternateModel: opts.altModel, - fastMode: opts.fast, - thinking: opts.thinking, - cliAuthMode: opts.cliAuthMode, - parityPack: opts.parityPack, - pack: opts.pack, - scenarioIds: opts.scenario, - enabledPluginIds: opts.enablePlugin, - concurrency: opts.concurrency, - allowFailures: opts.allowFailures, - image: opts.image, - cpus: opts.cpus, - memory: opts.memory, - disk: opts.disk, - preflight: opts.preflight, - runtimePair: opts.runtimePair, - runtimeParityTier: opts.runtimeParityTier, - }); - }, - ); + .action(async (opts: QaSuiteCliOptions) => { + await runQaSuiteCliCommand({ + repoRoot: opts.repoRoot, + outputDir: opts.outputDir, + transportId: opts.transport, + runner: opts.runner, + providerMode: opts.providerMode, + primaryModel: opts.model, + alternateModel: opts.altModel, + fastMode: opts.fast, + thinking: opts.thinking, + cliAuthMode: opts.cliAuthMode, + parityPack: opts.parityPack, + pack: opts.pack, + scenarioIds: opts.scenario, + enabledPluginIds: opts.enablePlugin, + concurrency: opts.concurrency, + allowFailures: opts.allowFailures, + image: opts.image, + cpus: opts.cpus, + memory: opts.memory, + disk: opts.disk, + preflight: opts.preflight, + runtimePair: opts.runtimePair, + runtimeParityTier: opts.runtimeParityTier, + }); + }); qa.command("parity-report") .description("Write either a model-axis parity gate report or a runtime-axis parity report") diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 17033f37a27a..4b7beaf867a5 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -8,7 +8,8 @@ const { runQaFlowSuite, runQaTestFileScenarios } = vi.hoisted(() => ({ runQaTestFileScenarios: vi.fn(), })); -vi.mock("./suite.js", () => ({ +vi.mock("./suite.js", async (importOriginal) => ({ + ...(await importOriginal()), runQaFlowSuite, })); @@ -27,26 +28,72 @@ async function makeTempRepo(prefix: string) { return repoRoot; } +async function writeEvidence(pathLocal: string) { + await fs.mkdir(path.dirname(pathLocal), { recursive: true }); + await fs.writeFile( + pathLocal, + `${JSON.stringify( + { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + entries: [], + }, + null, + 2, + )}\n`, + "utf8", + ); +} + describe("qa suite runtime launcher", () => { beforeEach(() => { runQaFlowSuite.mockReset(); runQaTestFileScenarios.mockReset(); - runQaFlowSuite.mockResolvedValue({ - outputDir: "/tmp/qa-flow", - evidencePath: "/tmp/qa-flow/qa-evidence.json", - reportPath: "/tmp/qa-flow/qa-suite-report.md", - summaryPath: "/tmp/qa-flow/qa-suite-summary.json", - report: "# QA Suite Report\n", - scenarios: [], - watchUrl: "http://127.0.0.1:43124", - }); - runQaTestFileScenarios.mockResolvedValue({ - outputDir: "/tmp/qa-test-file", - executionKind: "playwright", - reportPath: "/tmp/qa-test-file/qa-playwright-report.md", - evidencePath: "/tmp/qa-test-file/qa-evidence.json", - results: [{ status: "pass" }], + runQaFlowSuite.mockImplementation(async (params: { outputDir?: string } | undefined) => { + const outputDir = params?.outputDir ?? "/tmp/qa-flow"; + const evidencePath = path.join(outputDir, "qa-evidence.json"); + await writeEvidence(evidencePath); + return { + outputDir, + evidencePath, + reportPath: path.join(outputDir, "qa-suite-report.md"), + summaryPath: path.join(outputDir, "qa-suite-summary.json"), + report: "# QA Suite Report\n", + scenarios: [ + { + name: "channel-chat-baseline", + status: "pass", + steps: [], + }, + ], + watchUrl: "http://127.0.0.1:43124", + }; }); + runQaTestFileScenarios.mockImplementation( + async (params: { + outputDir: string; + scenarios: Array<{ id: string; execution: { kind: "vitest" | "playwright" } }>; + }) => { + const [scenario] = params.scenarios; + if (!scenario) { + throw new Error("expected scenario"); + } + const evidencePath = path.join(params.outputDir, "qa-evidence.json"); + await writeEvidence(evidencePath); + return { + outputDir: params.outputDir, + executionKind: scenario.execution.kind, + evidencePath, + results: params.scenarios.map((scenarioItem) => ({ + durationMs: 1, + logPath: path.join(params.outputDir, `${scenarioItem.id}.log`), + scenario: scenarioItem, + status: "pass", + })), + }; + }, + ); }); afterEach(async () => { @@ -88,9 +135,22 @@ describe("qa suite runtime launcher", () => { }); expect(result).toMatchObject({ - executionKind: "playwright", + executionKind: "suite", result: { - evidencePath: "/tmp/qa-test-file/qa-evidence.json", + evidencePath: path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-test", + "qa-evidence.json", + ), + summaryPath: path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-test", + "qa-suite-summary.json", + ), }, }); expect(runQaFlowSuite).not.toHaveBeenCalled(); @@ -98,7 +158,7 @@ describe("qa suite runtime launcher", () => { const [call] = runQaTestFileScenarios.mock.calls[0] ?? []; expect(call).toMatchObject({ repoRoot, - outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-test"), + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-test", "playwright"), providerMode: "mock-openai", primaryModel: "mock-openai/gpt-5.5", }); @@ -110,16 +170,53 @@ describe("qa suite runtime launcher", () => { ).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]); }); - it("rejects mixed flow and Vitest/Playwright scenarios", async () => { - await expect( - runQaSuite({ - repoRoot: process.cwd(), - scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"], - }), - ).rejects.toThrow("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios"); + it("runs mixed flow and Vitest/Playwright scenarios as one suite", async () => { + const repoRoot = await makeTempRepo("qa-suite-mixed-"); + const result = await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mixed", + scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"], + }); - expect(runQaFlowSuite).not.toHaveBeenCalled(); - expect(runQaTestFileScenarios).not.toHaveBeenCalled(); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "mixed"); + expect(result).toMatchObject({ + executionKind: "suite", + result: { + evidencePath: path.join(outputDir, "qa-evidence.json"), + summaryPath: path.join(outputDir, "qa-suite-summary.json"), + }, + }); + expect(runQaFlowSuite).toHaveBeenCalledWith( + expect.objectContaining({ + outputDir: path.join(outputDir, "flow"), + scenarioIds: ["channel-chat-baseline"], + }), + ); + expect(runQaTestFileScenarios).toHaveBeenCalledWith( + expect.objectContaining({ + outputDir: path.join(outputDir, "playwright"), + }), + ); + await expect(fs.access(path.join(outputDir, "qa-suite-summary.json"))).resolves.toBeUndefined(); + await expect(fs.access(path.join(outputDir, "qa-evidence.json"))).resolves.toBeUndefined(); + const summary = JSON.parse( + await fs.readFile(path.join(outputDir, "qa-suite-summary.json"), "utf8"), + ) as { + run?: { scenarioIds?: unknown }; + scenarios?: Array<{ details?: unknown; name?: unknown; status?: unknown }>; + }; + expect(summary.run?.scenarioIds).toEqual([ + "channel-chat-baseline", + "control-ui-chat-flow-playwright", + ]); + expect(summary.scenarios).toMatchObject([ + { name: "channel-chat-baseline", status: "pass" }, + { name: "Control UI chat flow Playwright coverage", status: "pass" }, + ]); + expect(JSON.stringify(summary)).not.toContain(repoRoot); + expect(summary.scenarios?.[1]?.details).toContain( + "log=.artifacts/qa-e2e/mixed/playwright/control-ui-chat-flow-playwright.log", + ); }); it("rejects runtime-pair requests for Vitest/Playwright scenarios", async () => { diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index e221aec121b5..7fee0a1d694d 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -1,10 +1,30 @@ // Qa Lab plugin module implements suite launch behavior. +import fs from "node:fs/promises"; import path from "node:path"; +import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime"; +import { toRepoRelativePath } from "./cli-paths.js"; +import { + QA_EVIDENCE_FILENAME, + QA_EVIDENCE_SUMMARY_KIND, + QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + validateQaEvidenceSummaryJson, + type QaEvidenceSummaryJson, +} from "./evidence-summary.js"; +import { isQaFastModeEnabled } from "./model-selection.js"; import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js"; import { defaultQaModelForMode, normalizeQaProviderMode } from "./run-config.js"; -import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; -import { resolveQaSuiteOutputDir } from "./suite-planning.js"; -import type { QaSuiteResult, QaSuiteRunParams } from "./suite.js"; +import { + readQaBootstrapScenarioCatalog, + type QaSeedScenarioWithSource, +} from "./scenario-catalog.js"; +import { normalizeQaSuiteConcurrency, resolveQaSuiteOutputDir } from "./suite-planning.js"; +import { + buildQaSuiteSummaryJson, + type QaSuiteResult, + type QaSuiteRunParams, + type QaSuiteScenarioResult, + type QaSuiteSummaryJson, +} from "./suite.js"; import { isQaTestFileScenario, runQaTestFileScenarios, @@ -19,8 +39,28 @@ export type QaSuiteRuntimeResult = result: QaSuiteResult; } | { - executionKind: QaTestFileExecutionKind; - result: QaTestFileScenarioRunResult; + executionKind: "suite"; + result: QaUnifiedSuiteResult; + }; + +export type QaUnifiedSuiteResult = { + evidencePath: string; + outputDir: string; + report: string; + reportPath: string; + scenarios: QaSuiteScenarioResult[]; + summaryPath: string; +}; + +type QaSuiteExecutionPlan = + | { + kind: "flow"; + } + | { + kind: "unified"; + scenarios: QaSeedScenarioWithSource[]; + flowScenarios: QaSeedScenarioWithSource[]; + testFileScenariosByKind: Map; }; async function loadQaLabServerRuntime() { @@ -42,25 +82,34 @@ function resolveRequestedScenarios(params: { }); } -function resolveTestFileScenariosForSuiteDispatch( - params: QaSuiteRunParams | undefined, -): QaTestFileScenario[] | null { +function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan { const scenarioIds = params?.scenarioIds ?? []; if (scenarioIds.length === 0) { - return null; + return { kind: "flow" }; } const selectedScenarios = resolveRequestedScenarios({ scenarioIds, scenarios: readQaBootstrapScenarioCatalog().scenarios, }); - const testFileScenarios = selectedScenarios.filter(isQaTestFileScenario); - if (testFileScenarios.length === 0) { - return null; + const flowScenarios = selectedScenarios.filter((scenario) => !isQaTestFileScenario(scenario)); + const testFileScenariosByKind = new Map(); + for (const scenario of selectedScenarios) { + if (!isQaTestFileScenario(scenario)) { + continue; + } + const scenarios = testFileScenariosByKind.get(scenario.execution.kind) ?? []; + scenarios.push(scenario); + testFileScenariosByKind.set(scenario.execution.kind, scenarios); } - if (testFileScenarios.length !== selectedScenarios.length) { - throw new Error("qa suite cannot mix execution.kind: flow with Vitest/Playwright scenarios."); + if (testFileScenariosByKind.size === 0) { + return { kind: "flow" }; } - return testFileScenarios; + return { + kind: "unified", + scenarios: selectedScenarios, + flowScenarios, + testFileScenariosByKind, + }; } async function runQaTestFileSuiteFromRuntime(params: { @@ -90,16 +139,241 @@ async function runQaTestFileSuiteFromRuntime(params: { }); } -export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise { - const runParams = args[0]; - const testFileScenarios = resolveTestFileScenariosForSuiteDispatch(runParams); - if (testFileScenarios) { +function rejectFlowOnlySuiteOptionsForUnifiedRun(runParams: QaSuiteRunParams | undefined) { + if (runParams?.runtimePair) { + throw new Error("--runtime-pair requires execution.kind: flow scenarios."); + } + if (runParams?.forcedRuntime) { + throw new Error("forced runtime execution requires execution.kind: flow scenarios."); + } + if (runParams?.captureRuntimeParityCell) { + throw new Error("runtime parity capture requires execution.kind: flow scenarios."); + } +} + +function suitePartitionOutputDir(outputDir: string, kind: "flow" | QaTestFileExecutionKind) { + return path.join(outputDir, kind); +} + +async function readQaSuiteEvidenceSummary(evidencePath: string) { + return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(evidencePath, "utf8"))); +} + +function mergeQaEvidenceSummaries(params: { + evidenceSummaries: readonly QaEvidenceSummaryJson[]; + generatedAt: string; +}) { + return validateQaEvidenceSummaryJson({ + kind: QA_EVIDENCE_SUMMARY_KIND, + schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + generatedAt: params.generatedAt, + entries: params.evidenceSummaries.flatMap((summary) => summary.entries), + }); +} + +function testFileScenarioResultToSuiteScenario( + result: QaTestFileScenarioRunResult["results"][number], + repoRoot: string, +): QaSuiteScenarioResult { + const suiteStatus = result.status === "pass" ? "pass" : "fail"; + const stepStatus = result.status === "skipped" ? "skip" : suiteStatus; + const logPath = toRepoRelativePath(repoRoot, result.logPath); + const details = [ + `execution.kind=${result.scenario.execution.kind}`, + `execution.path=${result.scenario.execution.path}`, + `log=${logPath}`, + ...(result.failureMessage ? [`failure=${result.failureMessage}`] : []), + ].join("\n"); + return { + name: result.scenario.title, + status: suiteStatus, + details, + steps: [ + { + name: `Run ${result.scenario.execution.kind} test file`, + status: stepStatus, + details, + }, + ], + }; +} + +function renderUnifiedQaSuiteReport(params: { + finishedAt: Date; + scenarios: readonly QaSuiteScenarioResult[]; + startedAt: Date; +}) { + return renderQaMarkdownReport({ + title: "OpenClaw QA Scenario Suite", + startedAt: params.startedAt, + finishedAt: params.finishedAt, + checks: [], + scenarios: params.scenarios.map((scenario) => ({ + name: scenario.name, + status: scenario.status, + details: scenario.details, + steps: scenario.steps, + })) satisfies QaReportScenario[], + }); +} + +async function writeUnifiedQaSuiteArtifacts(params: { + alternateModel: string; + concurrency: number; + evidence: QaEvidenceSummaryJson; + fastMode: boolean; + finishedAt: Date; + outputDir: string; + primaryModel: string; + providerMode: ReturnType; + scenarioIds: readonly string[]; + scenarios: readonly QaSuiteScenarioResult[]; + startedAt: Date; +}) { + await fs.mkdir(params.outputDir, { recursive: true }); + const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); + const reportPath = path.join(params.outputDir, "qa-suite-report.md"); + const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); + const report = renderUnifiedQaSuiteReport({ + finishedAt: params.finishedAt, + scenarios: params.scenarios, + startedAt: params.startedAt, + }); + const summary = buildQaSuiteSummaryJson({ + alternateModel: params.alternateModel, + concurrency: params.concurrency, + evidence: params.evidence, + fastMode: params.fastMode, + finishedAt: params.finishedAt, + primaryModel: params.primaryModel, + providerMode: params.providerMode, + scenarioIds: params.scenarioIds, + scenarios: [...params.scenarios], + startedAt: params.startedAt, + }) satisfies QaSuiteSummaryJson; + await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8"); + await fs.writeFile(reportPath, report, "utf8"); + await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + return { + evidencePath, + outputDir: params.outputDir, + report, + reportPath, + scenarios: [...params.scenarios], + summaryPath, + } satisfies QaUnifiedSuiteResult; +} + +async function runUnifiedQaSuite(params: { + plan: Extract; + runParams: QaSuiteRunParams | undefined; +}): Promise { + rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams); + const startedAt = new Date(); + const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd()); + const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir); + const providerMode = normalizeQaProviderMode( + params.runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE, + ); + const primaryModel = + params.runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const alternateModel = + params.runParams?.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const fastMode = + typeof params.runParams?.fastMode === "boolean" + ? params.runParams.fastMode + : isQaFastModeEnabled({ primaryModel, alternateModel }); + const concurrency = normalizeQaSuiteConcurrency( + params.runParams?.concurrency, + params.plan.scenarios.length, + ); + const evidenceSummaries: QaEvidenceSummaryJson[] = []; + const scenarioResultsById = new Map(); + if (params.plan.flowScenarios.length > 0) { + const flowResult = await runQaFlowSuiteFromRuntime({ + ...params.runParams, + outputDir: suitePartitionOutputDir(outputDir, "flow"), + providerMode, + primaryModel, + alternateModel, + fastMode, + scenarioIds: params.plan.flowScenarios.map((scenario) => scenario.id), + }); + for (const [index, scenario] of params.plan.flowScenarios.entries()) { + const result = flowResult.scenarios[index]; + if (result) { + scenarioResultsById.set(scenario.id, result); + } + } + evidenceSummaries.push(await readQaSuiteEvidenceSummary(flowResult.evidencePath)); + } + for (const [kind, testFileScenarios] of params.plan.testFileScenariosByKind) { const result = await runQaTestFileSuiteFromRuntime({ - runParams, + runParams: { + ...params.runParams, + outputDir: suitePartitionOutputDir(outputDir, kind), + providerMode, + primaryModel, + scenarioIds: testFileScenarios.map((scenario) => scenario.id), + }, scenarios: testFileScenarios, }); + for (const scenarioResult of result.results) { + scenarioResultsById.set( + scenarioResult.scenario.id, + testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot), + ); + } + evidenceSummaries.push(await readQaSuiteEvidenceSummary(result.evidencePath)); + } + const finishedAt = new Date(); + const evidence = mergeQaEvidenceSummaries({ + evidenceSummaries, + generatedAt: finishedAt.toISOString(), + }); + const scenarios = params.plan.scenarios.map((scenario) => { + const result = scenarioResultsById.get(scenario.id); + if (result) { + return result; + } return { - executionKind: result.executionKind, + name: scenario.title, + status: "fail", + details: "suite partition returned no scenario result", + steps: [ + { + name: "suite partition", + status: "fail", + details: "suite partition returned no scenario result", + }, + ], + } satisfies QaSuiteScenarioResult; + }); + return await writeUnifiedQaSuiteArtifacts({ + alternateModel, + concurrency, + evidence, + fastMode, + finishedAt, + outputDir, + primaryModel, + providerMode, + scenarioIds: params.plan.scenarios.map((scenario) => scenario.id), + scenarios, + startedAt, + }); +} + +export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise { + const runParams = args[0]; + const plan = resolveSuiteExecutionPlan(runParams); + if (plan.kind === "unified") { + const result = await runUnifiedQaSuite({ + runParams, + plan, + }); + return { + executionKind: "suite", result, }; } diff --git a/extensions/qa-lab/src/suite-planning.ts b/extensions/qa-lab/src/suite-planning.ts index f20eae7fd3eb..9631ae964280 100644 --- a/extensions/qa-lab/src/suite-planning.ts +++ b/extensions/qa-lab/src/suite-planning.ts @@ -29,7 +29,7 @@ function normalizeQaConfigString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } -function scenarioMatchesLiveLane(params: { +function scenarioMatchesQaProviderLane(params: { scenario: QaSeedScenario; primaryModel: string; providerMode: QaProviderMode; @@ -99,7 +99,7 @@ function selectQaFlowSuiteScenarios(params: { return params.scenarios.filter( (scenario) => scenario.execution.kind === "flow" && - scenarioMatchesLiveLane({ + scenarioMatchesQaProviderLane({ scenario, providerMode: params.providerMode, primaryModel: params.primaryModel, @@ -282,6 +282,7 @@ export { resolveQaSuiteWorkerStartStaggerMs, resolveQaSuiteOutputDir, scenarioRequiresControlUi, + scenarioMatchesQaProviderLane, selectQaFlowSuiteScenarios, shouldUseIsolatedQaSuiteScenarioWorkers, splitModelRef, diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index a81a048b71a1..bd7c39e3bd23 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -131,11 +131,6 @@ describe("qa test file scenario runner", () => { execution: { runner: "playwright", artifacts: [ - { - kind: "report", - path: ".artifacts/qa-e2e/scenario-playwright/qa-playwright-report.md", - source: "playwright", - }, { kind: "log", path: ".artifacts/qa-e2e/scenario-playwright/scenario-playwright.log", @@ -147,7 +142,6 @@ describe("qa test file scenario runner", () => { status: "pass", }, }); - expect(await fs.readFile(result.reportPath, "utf8")).toContain("Evidence summary"); }); it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => { @@ -203,11 +197,6 @@ describe("qa test file scenario runner", () => { execution: { runner: "vitest", artifacts: [ - { - kind: "report", - path: ".artifacts/qa-e2e/scenario-vitest/qa-vitest-report.md", - source: "vitest", - }, { kind: "log", path: ".artifacts/qa-e2e/scenario-vitest/scenario-vitest.log", diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index 0f38146563f7..20c6617a7632 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -69,15 +69,12 @@ export type QaTestFileScenarioRunResult = { evidencePath: string; executionKind: QaTestFileExecutionKind; outputDir: string; - reportPath: string; results: QaTestFileScenarioResult[]; }; type QaTestFileRunnerDefinition = { buildEvidenceSummary: typeof buildVitestEvidenceSummary; buildSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[]; - reportFilename: string; - reportTitle: string; }; export function isQaTestFileScenario( @@ -121,14 +118,10 @@ const testFileRunnerDefinitions: Record ({ - kind: "log", - path: toRepoRelativePath(params.repoRoot, result.logPath), - })), - ]; + return params.results.map((result) => ({ + kind: "log", + path: toRepoRelativePath(params.repoRoot, result.logPath), + })); } -function renderTestFileScenarioReport(params: { - evidencePath: string; - generatedAt: string; - repoRoot: string; - results: readonly QaTestFileScenarioResult[]; - title: string; -}) { - const lines = [ - `# ${params.title}`, - "", - `Generated at: ${params.generatedAt}`, - `Evidence summary: ${toRepoRelativePath(params.repoRoot, params.evidencePath)}`, - "", - "## Results", - "", - ]; - for (const result of params.results) { - const logPath = toRepoRelativePath(params.repoRoot, result.logPath); - lines.push( - `- ${result.scenario.id}: ${result.status}`, - ` - kind: ${result.scenario.execution.kind}`, - ` - path: ${result.scenario.execution.path}`, - ` - durationMs: ${Math.round(result.durationMs)}`, - ` - log: ${logPath}`, - ); - if (result.failureMessage) { - lines.push(` - failure: ${result.failureMessage.split("\n")[0]}`); - } - } - return `${lines.join("\n")}\n`; -} - -async function writeTestFileEvidenceFiles(params: { +async function writeTestFileEvidenceFile(params: { evidence: unknown; - generatedAt: string; outputDir: string; - reportFilename: string; - reportTitle: string; - repoRoot: string; - results: readonly QaTestFileScenarioResult[]; -}): Promise> { +}): Promise> { const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); - const reportPath = path.join(params.outputDir, params.reportFilename); await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8"); - const report = renderTestFileScenarioReport({ - evidencePath, - generatedAt: params.generatedAt, - repoRoot: params.repoRoot, - results: params.results, - title: params.reportTitle, - }); - await fs.writeFile(reportPath, report, "utf8"); await assertQaTestFileArtifactWritten("evidence", evidencePath); - await assertQaTestFileArtifactWritten("report", reportPath); - return { evidencePath, reportPath }; + return { evidencePath }; } -async function assertQaTestFileArtifactWritten(kind: "evidence" | "report", filePath: string) { +async function assertQaTestFileArtifactWritten(kind: "evidence", filePath: string) { try { await fs.access(filePath); } catch (error) { @@ -378,7 +320,6 @@ export async function runQaTestFileScenarios( if (!kind) { throw new Error("qa suite found no Vitest or Playwright scenarios to run."); } - const definition = testFileRunnerDefinitions[kind]; await fs.mkdir(params.outputDir, { recursive: true }); const runCommand = params.runCommand ?? runQaScenarioCommand; const env = { @@ -398,9 +339,7 @@ export async function runQaTestFileScenarios( ); } const generatedAt = new Date().toISOString(); - const reportPath = path.join(params.outputDir, definition.reportFilename); const artifactPaths = buildScenarioArtifactPaths({ - reportPath, repoRoot: params.repoRoot, results, }); @@ -413,14 +352,9 @@ export async function runQaTestFileScenarios( providerMode: params.providerMode, results, }); - const paths = await writeTestFileEvidenceFiles({ + const paths = await writeTestFileEvidenceFile({ evidence, - generatedAt, outputDir: params.outputDir, - reportFilename: definition.reportFilename, - reportTitle: definition.reportTitle, - repoRoot: params.repoRoot, - results, }); return { ...paths, diff --git a/src/cli/profile.test.ts b/src/cli/profile.test.ts index a463b3e0f347..2bc2e12c1ccc 100644 --- a/src/cli/profile.test.ts +++ b/src/cli/profile.test.ts @@ -111,6 +111,128 @@ describe("parseCliProfileArgs", () => { expect(res.argv).toEqual(["node", "openclaw", "--no-color", "qa", "matrix", "--profile=fast"]); }); + it("parses qa run --profile smoke-ci as a root profile", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "qa", + "run", + "--profile", + "smoke-ci", + "--category", + "agent-runtime-and-provider-execution.agent-turn-execution", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBe("smoke-ci"); + expect(res.argv).toEqual([ + "node", + "openclaw", + "qa", + "run", + "--category", + "agent-runtime-and-provider-execution.agent-turn-execution", + ]); + }); + + it("parses qa run --profile=release self-check invocations as root profiles", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "qa", + "run", + "--profile=release", + "--output", + "qa-report.md", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBe("release"); + expect(res.argv).toEqual(["node", "openclaw", "qa", "run", "--output", "qa-report.md"]); + }); + + it("preserves qa run --qa-profile for the command parser", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "smoke-ci", + "--surface", + "agent-runtime-and-provider-execution", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBeNull(); + expect(res.argv).toEqual([ + "node", + "openclaw", + "qa", + "run", + "--qa-profile", + "smoke-ci", + "--surface", + "agent-runtime-and-provider-execution", + ]); + }); + + it("parses arbitrary qa run --profile values as root profiles", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "qa", + "run", + "--profile", + "work", + "--output", + "qa-report.md", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBe("work"); + expect(res.argv).toEqual(["node", "openclaw", "qa", "run", "--output", "qa-report.md"]); + }); + + it("parses arbitrary qa run --profile= values as root profiles", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "qa", + "run", + "--profile=work", + "--output", + "qa-report.md", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBe("work"); + expect(res.argv).toEqual(["node", "openclaw", "qa", "run", "--output", "qa-report.md"]); + }); + + it("still parses root --profile before qa run", () => { + const res = parseCliProfileArgs([ + "node", + "openclaw", + "--profile", + "work", + "qa", + "run", + "--qa-profile", + "smoke-ci", + ]); + if (!res.ok) { + throw new Error(res.error); + } + expect(res.profile).toBe("work"); + expect(res.argv).toEqual(["node", "openclaw", "qa", "run", "--qa-profile", "smoke-ci"]); + }); + it("still parses root --profile before Matrix QA", () => { const res = parseCliProfileArgs([ "node", diff --git a/src/cli/profile.ts b/src/cli/profile.ts index 63c0a11d188c..92a22d42a64c 100644 --- a/src/cli/profile.ts +++ b/src/cli/profile.ts @@ -5,7 +5,6 @@ import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; -import { isValueToken } from "../infra/cli-root-options.js"; import { resolveRequiredHomeDir } from "../infra/home-dir.js"; import { resolveCliArgvInvocation } from "./argv-invocation.js"; import { isValidProfileName } from "./profile-utils.js"; @@ -16,11 +15,6 @@ type CliProfileParseResult = | { ok: true; profile: string | null; argv: string[] } | { ok: false; error: string }; -function isCommandLocalProfileOption(out: string[]): boolean { - const [primary, secondary] = resolveCliArgvInvocation(out).commandPath; - return primary === "qa" && secondary === "matrix"; -} - export function parseCliProfileArgs(argv: string[]): CliProfileParseResult { // Root profile flags are stripped before Commander sees argv, except command-local cases. let profile: string | null = null; @@ -41,19 +35,19 @@ export function parseCliProfileArgs(argv: string[]): CliProfileParseResult { } if (arg === "--profile" || arg.startsWith("--profile=")) { - if (isCommandLocalProfileOption(out)) { + const next = args[index + 1]; + const { value, consumedNext } = takeCliRootOptionValue(arg, next); + const [primary, secondary] = resolveCliArgvInvocation(out).commandPath; + if (primary === "qa" && secondary === "matrix") { out.push(arg); - if (arg === "--profile" && isValueToken(args[index + 1])) { - out.push(args[index + 1]); - return { kind: "handled", consumedNext: true }; + if (consumedNext) { + out.push(next); } - return { kind: "handled" }; + return { kind: "handled", consumedNext }; } if (sawDev) { return { kind: "error", error: "Cannot combine --dev with --profile" }; } - const next = args[index + 1]; - const { value, consumedNext } = takeCliRootOptionValue(arg, next); if (!value) { return { kind: "error", error: "--profile requires a value" }; }