From 591313e80a5e419e8a4d824cd8e8a6ae6689fb79 Mon Sep 17 00:00:00 2001 From: Colin Johnson Date: Wed, 17 Jun 2026 18:09:25 -0400 Subject: [PATCH] qa-lab: support script-backed evidence scenarios (#94276) * qa: add script scenario execution kind * fix(qa-lab): carry suite profile into script producer evidence and simplify artifact path resolution * fix(qa-lab): keep out-of-repo producer artifacts absolute to avoid ../ traversal refs --------- Co-authored-by: Dallin Romney --- docs/concepts/qa-e2e-automation.md | 7 +- extensions/qa-lab/src/coverage-report.test.ts | 37 +- extensions/qa-lab/src/coverage-report.ts | 7 +- extensions/qa-lab/src/evidence-summary.ts | 16 +- extensions/qa-lab/src/scenario-catalog.ts | 4 + extensions/qa-lab/src/scorecard-taxonomy.ts | 2 +- .../qa-lab/src/suite-launch.runtime.test.ts | 2 +- .../src/test-file-scenario-runner.test.ts | 487 +++++++++++++++++- .../qa-lab/src/test-file-scenario-runner.ts | 263 +++++++++- qa/scenarios/index.yaml | 6 +- 10 files changed, 811 insertions(+), 20 deletions(-) diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 666f1f2f9b1e..c831e409bf7f 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -942,7 +942,12 @@ Every `qa suite` run writes top-level `qa-evidence.json`, `qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected scenario set. Scenarios that declare `execution.kind: vitest` or `execution.kind: playwright` run the matching test path and also write -per-scenario logs. When `qa suite` is reached through +per-scenario logs. Scenarios that declare `execution.kind: script` run the +evidence producer at `execution.path` through `node --import tsx` (with +`${outputDir}` and `${scenarioId}` expanded in `execution.args`); the producer +writes its own `qa-evidence.json`, whose entries are imported into the suite +output and whose artifact paths are resolved relative to that producer +`qa-evidence.json`. When `qa suite` is reached through `qa run --qa-profile`, the same `qa-evidence.json` also includes the profile scorecard summary for the selected taxonomy categories. Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test. diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index 4707e8ccb260..3b56fadd6fb2 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -67,11 +67,13 @@ function scenarioWithCoverage(params: { primary?: readonly string[]; secondary?: readonly string[]; sourcePath?: string; - executionKind?: "flow" | "vitest" | "playwright"; + executionKind?: "flow" | "script" | "vitest" | "playwright"; executionPath?: string; }): QaSeedScenarioWithSource { const execution = - params.executionKind === "vitest" || params.executionKind === "playwright" + params.executionKind === "script" || + params.executionKind === "vitest" || + params.executionKind === "playwright" ? { kind: params.executionKind, path: params.executionPath ?? "src/test.test.ts", @@ -328,6 +330,37 @@ describe("qa coverage report", () => { ]); }); + it("uses script producer evidence as coverage fulfillment", () => { + const report = buildQaScorecardTaxonomyReport({ + taxonomy: testMaturityTaxonomy({ + categoryId: TEST_BROWSER_CATEGORY_ID, + coverageIds: [TEST_BROWSER_COVERAGE_ID], + }), + repoRoot: process.cwd(), + scenarios: [ + scenarioWithCoverage({ + primary: [TEST_BROWSER_COVERAGE_ID], + sourcePath: "qa/scenarios/ui/script-evidence-producer.yaml", + executionKind: "script", + executionPath: "scripts/check-no-conflict-markers.mjs", + }), + ], + }); + + expect(report.validationIssues).toStrictEqual([]); + expect(report.fulfilledCategoryCount).toBe(1); + expect(report.fulfilledFeatureCount).toBe(1); + expect(report.categories[0]?.evidence).toStrictEqual([ + { + coverageId: TEST_BROWSER_COVERAGE_ID, + kind: "script", + path: "scripts/check-no-conflict-markers.mjs", + role: "primary", + scenarioRefs: ["qa/scenarios/ui/script-evidence-producer.yaml"], + }, + ]); + }); + it("reports profile membership refs missing from taxonomy categories", () => { const report = buildQaScorecardTaxonomyReport({ taxonomy: testMaturityTaxonomy({ diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index c3b2fa44a31c..5c12bc5004e9 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -465,7 +465,12 @@ function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) { group.push(match); groups.set(match.executionKind, group); } - const executionOrder: QaScenarioSearchMatch["executionKind"][] = ["flow", "vitest", "playwright"]; + const executionOrder: QaScenarioSearchMatch["executionKind"][] = [ + "flow", + "script", + "vitest", + "playwright", + ]; return executionOrder.flatMap((executionKind) => { const group = groups.get(executionKind); return group && group.length > 0 ? [{ executionKind, matches: group }] : []; diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts index 7c8d38582850..9212af798577 100644 --- a/extensions/qa-lab/src/evidence-summary.ts +++ b/extensions/qa-lab/src/evidence-summary.ts @@ -349,7 +349,7 @@ function uniqueSortedStrings(values: readonly (string | undefined)[]) { ); } -function resolveQaEvidenceProfile(params: { +export function resolveQaEvidenceProfile(params: { env?: NodeJS.ProcessEnv; explicit?: QaEvidenceProfile; }) { @@ -705,6 +705,20 @@ export function buildPlaywrightEvidenceSummary( }); } +export function buildScriptEvidenceSummary( + params: QaEvidenceBuildBase & { + targets: readonly QaEvidenceTestTargetInput[]; + results: readonly QaEvidenceTestResultInput[]; + }, +): QaEvidenceSummaryJson { + return buildTestRunnerEvidenceSummary({ + ...params, + defaultRunner: "script", + testKind: "script-test", + runner: params.runner ?? "script", + }); +} + export function buildLiveTransportEvidenceSummary( params: QaEvidenceBuildBase & { checks: readonly QaEvidenceLiveTransportCheckInput[]; diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 787954152502..5cee8c614301 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -73,6 +73,10 @@ const qaTestFileScenarioExecutionBaseSchema = z.object({ const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [ qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("vitest") }), qaTestFileScenarioExecutionBaseSchema.extend({ kind: z.literal("playwright") }), + qaTestFileScenarioExecutionBaseSchema.extend({ + kind: z.literal("script"), + args: z.array(z.string()).optional(), + }), ]); const qaScenarioExecutionSchema = z.union([ diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts index 7359c206cc94..b2cfde1ba2ab 100644 --- a/extensions/qa-lab/src/scorecard-taxonomy.ts +++ b/extensions/qa-lab/src/scorecard-taxonomy.ts @@ -90,7 +90,7 @@ const qaMaturityTaxonomySchema = z } }); -export type QaNativeCoverageEvidenceKind = "vitest" | "playwright"; +export type QaNativeCoverageEvidenceKind = "script" | "vitest" | "playwright"; export type QaScorecardEvidenceKind = QaNativeCoverageEvidenceKind | "qa-scenario"; export type QaScorecardEvidenceMode = z.infer; type QaCoverageEvidenceRole = z.infer; diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 8d56824e1629..d54006a165f4 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -74,7 +74,7 @@ describe("qa suite runtime launcher", () => { runQaTestFileScenarios.mockImplementation( async (params: { outputDir: string; - scenarios: Array<{ id: string; execution: { kind: "vitest" | "playwright" } }>; + scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>; }) => { const [scenario] = params.scenarios; if (!scenario) { diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index 8f5f13292fa4..12a0b41df325 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -12,7 +12,7 @@ import { const tempRoots: string[] = []; function makeTestFileScenario( - executionKind: "vitest" | "playwright", + executionKind: "script" | "vitest" | "playwright", pathLocal: string, ): QaSeedScenarioWithSource { return { @@ -35,6 +35,9 @@ function makeTestFileScenario( execution: { kind: executionKind, path: pathLocal, + ...(executionKind === "script" + ? { args: ["--once", "--artifact-base", "${outputDir}"] } + : {}), }, }; } @@ -204,4 +207,486 @@ describe("qa test file scenario runner", () => { }, }); }); + + it("runs script scenarios and imports producer QA evidence artifacts", async () => { + const repoRoot = await makeTempRepo("qa-script-scenario-"); + const commands: QaScenarioCommandExecution[] = []; + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async (command) => { + commands.push(command); + const scenarioArtifactBase = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script", + "scenario-script", + ); + const runRoot = path.join(scenarioArtifactBase, "run-1"); + await fs.mkdir(path.join(runRoot, "surfaces", "web-ui"), { recursive: true }); + await fs.writeFile(path.join(runRoot, "surfaces", "web-ui", "screenshot.png"), "png"); + await fs.writeFile( + path.join(runRoot, "qa-evidence.json"), + `${JSON.stringify( + { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "scripts/evidence-producer.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + execution: { + runner: "evidence-producer-script", + environment: { + ref: "scenario-ref", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "script-producer", + live: false, + model: { name: null, ref: null }, + fixture: "mocked-script-evidence", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [ + { + kind: "screenshot", + path: "surfaces/web-ui/screenshot.png", + source: "script-producer:web-ui:smoke", + }, + ], + }, + result: { status: "pass", timing: { wallMs: 1 } }, + }, + ], + }, + null, + 2, + )}\n`, + "utf8", + ); + await fs.writeFile( + path.join(scenarioArtifactBase, "latest-run.json"), + `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, + "utf8", + ); + return { + exitCode: 0, + stdout: "script pass\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.executionKind).toBe("script"); + expect(commands.map((command) => command.args)).toEqual([ + [ + "--import", + "tsx", + "scripts/evidence-producer.ts", + "--once", + "--artifact-base", + path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script", "scenario-script"), + ], + ]); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + }, + coverage: [ + { + id: "ui.control", + role: "primary", + }, + ], + execution: { + runner: "evidence-producer-script", + artifacts: [ + { + kind: "screenshot", + path: ".artifacts/qa-e2e/scenario-script/scenario-script/run-1/surfaces/web-ui/screenshot.png", + source: "script-producer:web-ui:smoke", + }, + ], + }, + result: { + status: "pass", + }, + }); + }); + + it("imports producer QA evidence artifacts from failed script scenarios", async () => { + const repoRoot = await makeTempRepo("qa-script-failed-scenario-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-failed"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + const scenarioArtifactBase = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-failed", + "scenario-script", + ); + const runRoot = path.join(scenarioArtifactBase, "run-1"); + await fs.mkdir(runRoot, { recursive: true }); + await fs.writeFile( + path.join(runRoot, "qa-evidence.json"), + `${JSON.stringify( + { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "scripts/evidence-producer.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + execution: { + runner: "evidence-producer-script", + environment: { + ref: "scenario-ref", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "script-producer", + live: false, + model: { name: null, ref: null }, + fixture: "failed-producer-evidence", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [], + }, + result: { + status: "fail", + failure: { + reason: "Script producer check failed.", + }, + timing: { wallMs: 1 }, + }, + }, + ], + }, + null, + 2, + )}\n`, + "utf8", + ); + await fs.writeFile( + path.join(scenarioArtifactBase, "latest-run.json"), + `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, + "utf8", + ); + return { + exitCode: 1, + stdout: "", + stderr: "script failed\n", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "fail", + failureMessage: "node exited with 1", + producerEvidence: { + entries: [ + { + test: { + id: "script-producer.web-ui.smoke", + }, + result: { + status: "fail", + }, + }, + ], + }, + }); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(2); + expect(evidence.entries[0]).toMatchObject({ + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + }, + result: { + status: "fail", + failure: { + reason: "Script producer check failed.", + }, + }, + }); + expect(evidence.entries[1]).toMatchObject({ + test: { + kind: "script-test", + id: "scenario-script", + source: { + path: "scripts/evidence-producer.ts", + }, + }, + result: { + status: "fail", + failure: { + reason: "node exited with 1", + }, + }, + }); + }); + + it("fails script scenario results when imported producer evidence fails", async () => { + const repoRoot = await makeTempRepo("qa-script-producer-fail-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-producer-fail"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + const scenarioArtifactBase = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-producer-fail", + "scenario-script", + ); + const runRoot = path.join(scenarioArtifactBase, "run-1"); + await fs.mkdir(runRoot, { recursive: true }); + await fs.writeFile( + path.join(runRoot, "qa-evidence.json"), + `${JSON.stringify( + { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "scripts/evidence-producer.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + execution: { + runner: "evidence-producer-script", + environment: { + ref: "scenario-ref", + os: "darwin", + nodeVersion: "v24.0.0", + }, + provider: { + id: "script-producer", + live: false, + model: { name: null, ref: null }, + fixture: "mocked-script-evidence", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [], + }, + result: { + status: "fail", + failure: { + reason: "Script producer check failed.", + }, + timing: { wallMs: 1 }, + }, + }, + ], + }, + null, + 2, + )}\n`, + "utf8", + ); + await fs.writeFile( + path.join(scenarioArtifactBase, "latest-run.json"), + `${JSON.stringify({ qaEvidence: path.join(runRoot, "qa-evidence.json") }, null, 2)}\n`, + "utf8", + ); + return { + exitCode: 0, + stdout: "script pass\n", + stderr: "", + }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + } as NodeJS.ProcessEnv, + }); + + expect(result.results[0]).toMatchObject({ + status: "fail", + failureMessage: "Script producer check failed.", + }); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + test: { + id: "script-producer.web-ui.smoke", + }, + result: { + status: "fail", + }, + }); + }); + + it("carries the suite profile into merged producer evidence", async () => { + const repoRoot = await makeTempRepo("qa-script-profile-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-profile"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + const scenarioOutputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-profile", + "scenario-script", + ); + await fs.mkdir(scenarioOutputDir, { recursive: true }); + await fs.writeFile( + path.join(scenarioOutputDir, "qa-evidence.json"), + `${JSON.stringify({ + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "scripts/evidence-producer.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + result: { status: "pass", timing: { wallMs: 1 } }, + }, + ], + })}\n`, + "utf8", + ); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { + OPENCLAW_QA_REF: "scenario-ref", + OPENCLAW_QA_PROFILE: "smoke-ci", + } as NodeJS.ProcessEnv, + }); + + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + expect(evidence.profile).toBe("smoke-ci"); + }); + + it("keeps producer artifacts outside the repo root absolute instead of emitting ../ paths", async () => { + const repoRoot = await makeTempRepo("qa-script-external-artifact-"); + const externalArtifact = path.join(os.tmpdir(), "qa-external-artifact.png"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-script-external"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("script", "scripts/evidence-producer.ts")], + runCommand: async () => { + const scenarioOutputDir = path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "scenario-script-external", + "scenario-script", + ); + await fs.mkdir(scenarioOutputDir, { recursive: true }); + await fs.writeFile( + path.join(scenarioOutputDir, "qa-evidence.json"), + `${JSON.stringify({ + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [ + { + test: { + kind: "script-producer-check", + id: "script-producer.web-ui.smoke", + title: "Script producer: web-ui smoke", + source: { path: "scripts/evidence-producer.ts" }, + }, + coverage: [{ id: "ui.control", role: "primary" }], + execution: { + runner: "evidence-producer-script", + environment: { ref: "scenario-ref", os: "darwin", nodeVersion: "v24.0.0" }, + provider: { + id: "script-producer", + live: false, + model: { name: null, ref: null }, + fixture: "mocked-script-evidence", + }, + packageSource: { kind: "source-checkout", sha: "abc123" }, + artifacts: [ + { + kind: "screenshot", + path: externalArtifact, + source: "script-producer:web-ui:smoke", + }, + ], + }, + result: { status: "pass", timing: { wallMs: 1 } }, + }, + ], + })}\n`, + "utf8", + ); + return { exitCode: 0, stdout: "script pass\n", stderr: "" }; + }, + env: { OPENCLAW_QA_REF: "scenario-ref" } as NodeJS.ProcessEnv, + }); + + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); + const artifactPath = evidence.entries[0]?.execution?.artifacts[0]?.path; + expect(artifactPath).toBe(path.normalize(externalArtifact)); + expect(artifactPath?.includes("..")).toBe(false); + }); }); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index 462db3bb5415..178beebbe78c 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -2,15 +2,18 @@ import { spawn } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { toRepoRelativePath } from "./cli-paths.js"; +import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js"; import { QaSuiteArtifactError } from "./errors.js"; import { buildPlaywrightEvidenceSummary, + buildScriptEvidenceSummary, buildVitestEvidenceSummary, QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND, QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, type QaEvidenceStatus, + type QaEvidenceSummaryJson, + resolveQaEvidenceProfile, validateQaEvidenceSummaryJson, } from "./evidence-summary.js"; import type { QaProviderMode } from "./providers/index.js"; @@ -19,10 +22,13 @@ import type { QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; import { shellQuote } from "./shell-quote.js"; export type QaTestFileScenario = QaSeedScenarioWithSource & { - execution: Extract; + execution: Extract< + QaSeedScenarioWithSource["execution"], + { kind: "script" | "vitest" | "playwright" } + >; }; -export type QaTestFileExecutionKind = "vitest" | "playwright"; +export type QaTestFileExecutionKind = "script" | "vitest" | "playwright"; export type QaTestFileScenarioRunParams = { evidenceMode?: QaScorecardEvidenceMode; @@ -61,7 +67,9 @@ type QaScenarioCommandStep = { type QaTestFileScenarioResult = { durationMs: number; failureMessage?: string; + includeFallbackEvidence?: boolean; logPath: string; + producerEvidence?: QaEvidenceSummaryJson; scenario: QaTestFileScenario; status: QaEvidenceStatus; }; @@ -75,13 +83,17 @@ export type QaTestFileScenarioRunResult = { type QaTestFileRunnerDefinition = { buildEvidenceSummary: typeof buildVitestEvidenceSummary; - buildSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[]; + buildSteps(scenario: QaTestFileScenario, context: { outputDir: string }): QaScenarioCommandStep[]; }; export function isQaTestFileScenario( scenario: QaSeedScenarioWithSource, ): scenario is QaTestFileScenario { - return scenario.execution.kind === "vitest" || scenario.execution.kind === "playwright"; + return ( + scenario.execution.kind === "vitest" || + scenario.execution.kind === "playwright" || + scenario.execution.kind === "script" + ); } function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] { @@ -115,7 +127,42 @@ function playwrightSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] ]; } +function replaceScriptArgTokens( + args: readonly string[] | undefined, + context: { outputDir: string; scenarioId: string }, +) { + return (args ?? []).map((arg) => + arg + .replaceAll("${outputDir}", context.outputDir) + .replaceAll("${scenarioId}", context.scenarioId), + ); +} + +function scriptSteps( + scenario: QaTestFileScenario, + context: { outputDir: string }, +): QaScenarioCommandStep[] { + const scenarioOutputDir = path.join(context.outputDir, scenario.id); + const scriptArgs = + scenario.execution.kind === "script" + ? replaceScriptArgTokens(scenario.execution.args, { + outputDir: scenarioOutputDir, + scenarioId: scenario.id, + }) + : []; + return [ + { + command: process.execPath, + args: ["--import", "tsx", scenario.execution.path, ...scriptArgs], + }, + ]; +} + const testFileRunnerDefinitions: Record = { + script: { + buildEvidenceSummary: buildScriptEvidenceSummary, + buildSteps: scriptSteps, + }, vitest: { buildEvidenceSummary: buildVitestEvidenceSummary, buildSteps: vitestSteps, @@ -230,16 +277,64 @@ async function runQaTestFileScenario(params: { scenario: QaTestFileScenario; }) { const definition = testFileRunnerDefinitions[params.scenario.execution.kind]; - return await runScenarioCommandSteps({ + const result = await runScenarioCommandSteps({ ...params, - steps: definition.buildSteps(params.scenario), + steps: definition.buildSteps(params.scenario, { outputDir: params.outputDir }), }); + if (params.scenario.execution.kind !== "script") { + return result; + } + const producerEvidenceResult = await readScriptProducerEvidence({ + outputDir: params.outputDir, + repoRoot: params.repoRoot, + scenario: params.scenario, + }); + if (!producerEvidenceResult.producerEvidence) { + return result; + } + if (result.status !== "pass") { + return { + ...result, + ...producerEvidenceResult, + includeFallbackEvidence: true, + }; + } + return { + ...result, + ...producerEvidenceResult, + ...statusFromProducerEvidence(producerEvidenceResult.producerEvidence), + }; +} + +function statusFromProducerEvidence( + producerEvidence: QaEvidenceSummaryJson | undefined, +): Pick { + if (!producerEvidence || producerEvidence.entries.length === 0) { + return { status: "pass" }; + } + const blockingEntry = producerEvidence.entries.find( + (entry) => entry.result.status === "fail" || entry.result.status === "blocked", + ); + if (blockingEntry) { + return { + failureMessage: + blockingEntry.result.failure?.reason ?? + `${blockingEntry.test.id} reported ${blockingEntry.result.status}`, + status: blockingEntry.result.status, + }; + } + if (producerEvidence.entries.every((entry) => entry.result.status === "skipped")) { + return { status: "skipped" }; + } + return { status: "pass" }; } function resolveTestFileExecutionKind(scenarios: readonly QaTestFileScenario[]) { const kinds = new Set(scenarios.map((scenario) => scenario.execution.kind)); if (kinds.size > 1) { - throw new Error("qa suite cannot mix Vitest and Playwright scenarios in one invocation."); + throw new Error( + "qa suite cannot mix script, Vitest, and Playwright scenarios in one invocation.", + ); } const [kind] = kinds; return kind; @@ -255,6 +350,55 @@ function buildTestFileEvidence(params: { evidenceMode?: QaScorecardEvidenceMode; env?: NodeJS.ProcessEnv; }) { + const producerEntries = params.results.flatMap( + (result) => result.producerEvidence?.entries ?? [], + ); + if (producerEntries.length > 0) { + const definition = testFileRunnerDefinitions[params.kind]; + const fallbackResults = params.results.filter( + (result) => !result.producerEvidence || result.includeFallbackEvidence, + ); + const evidenceMode = + params.evidenceMode ?? + (params.results.every((result) => result.producerEvidence?.evidenceMode === "slim") + ? "slim" + : "full"); + const fallbackEvidence = + fallbackResults.length > 0 + ? definition.buildEvidenceSummary({ + artifactPaths: params.artifactPaths, + evidenceMode, + env: params.env, + generatedAt: params.generatedAt, + primaryModel: params.primaryModel, + providerMode: params.providerMode, + targets: fallbackResults.map((result) => buildScenarioEvidenceTarget(result.scenario)), + results: fallbackResults.map((result) => ({ + id: result.scenario.id, + status: result.status, + durationMs: result.durationMs, + failureMessage: result.failureMessage, + })), + }) + : undefined; + return validateQaEvidenceSummaryJson({ + kind: QA_EVIDENCE_SUMMARY_KIND, + schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, + generatedAt: params.generatedAt, + evidenceMode, + profile: resolveQaEvidenceProfile({ env: params.env }), + entries: [ + ...producerEntries.map((entry) => { + if (evidenceMode !== "slim") { + return entry; + } + const { execution: _execution, ...withoutExecution } = entry; + return withoutExecution; + }), + ...(fallbackEvidence?.entries ?? []), + ], + }); + } const definition = testFileRunnerDefinitions[params.kind]; const evidence = definition.buildEvidenceSummary({ artifactPaths: params.artifactPaths, @@ -281,6 +425,107 @@ function buildTestFileEvidence(params: { }); } +async function readJsonFileIfExists(filePath: string): Promise { + let text: string; + try { + text = await fs.readFile(filePath, "utf8"); + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + (error as { code?: unknown }).code === "ENOENT" + ) { + return undefined; + } + throw error; + } + try { + return JSON.parse(text) as unknown; + } catch (error) { + throw new Error(`invalid JSON in ${filePath}: ${formatErrorMessage(error)}`, { cause: error }); + } +} + +// Producer artifact paths follow one convention: relative paths resolve against the +// qa-evidence.json directory, absolute paths are taken as-is. Paths under the repo root +// become repo-relative; paths outside it stay absolute so downstream consumers never see +// `../` segments that would read as path traversal. +function resolveScriptProducerArtifactPath(params: { + evidenceDir: string; + repoRoot: string; + artifactPath: string; +}) { + const absolutePath = path.isAbsolute(params.artifactPath) + ? params.artifactPath + : path.join(params.evidenceDir, params.artifactPath); + const repoRelativePath = toRepoRelativePath(params.repoRoot, absolutePath); + return isRepoRootRelativeRef(repoRelativePath) ? repoRelativePath : path.normalize(absolutePath); +} + +function normalizeScriptProducerEvidence(params: { + evidence: QaEvidenceSummaryJson; + evidencePath: string; + repoRoot: string; +}): QaEvidenceSummaryJson { + // Input is already validated by the caller; this only rewrites artifact path strings, + // so the transformed shape stays schema-valid without re-parsing. + const evidenceDir = path.dirname(params.evidencePath); + return { + ...params.evidence, + entries: params.evidence.entries.map((entry) => ({ + ...entry, + execution: entry.execution + ? { + ...entry.execution, + artifacts: entry.execution.artifacts.map((artifact) => ({ + ...artifact, + path: resolveScriptProducerArtifactPath({ + artifactPath: artifact.path, + evidenceDir, + repoRoot: params.repoRoot, + }), + })), + } + : undefined, + })), + }; +} + +async function readScriptProducerEvidence(params: { + outputDir: string; + repoRoot: string; + scenario: QaTestFileScenario; +}): Promise> { + const scenarioOutputDir = path.join(params.outputDir, params.scenario.id); + const latestRun = (await readJsonFileIfExists( + path.join(scenarioOutputDir, "latest-run.json"), + )) as { qaEvidence?: unknown } | undefined; + const candidates = [ + typeof latestRun?.qaEvidence === "string" ? latestRun.qaEvidence : undefined, + path.join(scenarioOutputDir, QA_EVIDENCE_FILENAME), + ].filter((candidate): candidate is string => Boolean(candidate)); + + for (const candidate of candidates) { + const evidencePath = path.isAbsolute(candidate) + ? candidate + : path.join(scenarioOutputDir, candidate); + const rawEvidence = await readJsonFileIfExists(evidencePath); + if (!rawEvidence) { + continue; + } + const evidence = validateQaEvidenceSummaryJson(rawEvidence); + return { + producerEvidence: normalizeScriptProducerEvidence({ + evidence, + evidencePath, + repoRoot: params.repoRoot, + }), + }; + } + return {}; +} + function buildScenarioArtifactPaths(params: { repoRoot: string; results: readonly QaTestFileScenarioResult[]; @@ -319,7 +564,7 @@ export async function runQaTestFileScenarios( const scenarios = params.scenarios.filter(isQaTestFileScenario); const kind = resolveTestFileExecutionKind(scenarios); if (!kind) { - throw new Error("qa suite found no Vitest or Playwright scenarios to run."); + throw new Error("qa suite found no script, Vitest, or Playwright scenarios to run."); } await fs.mkdir(params.outputDir, { recursive: true }); const runCommand = params.runCommand ?? runQaScenarioCommand; diff --git a/qa/scenarios/index.yaml b/qa/scenarios/index.yaml index 4eed664b3917..8ca79dc1a781 100644 --- a/qa/scenarios/index.yaml +++ b/qa/scenarios/index.yaml @@ -22,9 +22,9 @@ title: OpenClaw QA Scenario Pack # - prefer reusing an existing coverage ID over minting a scenario-shaped ID # - avoid copying the scenario title into coverage IDs # - use `pnpm openclaw qa coverage` to render the current inventory -# - use `scenario.execution.kind: vitest` or `scenario.execution.kind: playwright` -# plus `scenario.execution.path` for native test files that provide evidence without -# a top-level `flow` +# - use `scenario.execution.kind: vitest`, `playwright`, or `script` +# plus `scenario.execution.path` for native tests or evidence producers that +# provide evidence without a top-level `flow` # - use `runtimeParityTier` for runtime-pair gate membership: `standard`, # `optional`, `live-only`, or `soak` # - treat the old `coverage: ["id"]` / `coverage: - id` list shape as invalid