From 0a37f797f2092eafc86f63df31633d56bc2eeb66 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 02:41:39 +0200 Subject: [PATCH] fix(test): validate gauntlet qa summary counts --- scripts/lib/plugin-gateway-gauntlet.mjs | 41 +++++++++- test/scripts/plugin-gateway-gauntlet.test.ts | 80 ++++++++++++++++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/scripts/lib/plugin-gateway-gauntlet.mjs b/scripts/lib/plugin-gateway-gauntlet.mjs index 1930619bf46b..4ef415edc6ae 100644 --- a/scripts/lib/plugin-gateway-gauntlet.mjs +++ b/scripts/lib/plugin-gateway-gauntlet.mjs @@ -492,18 +492,53 @@ function validateQaSuiteSummary(summary) { if ( !summary.counts || typeof summary.counts !== "object" || - !Number.isFinite(summary.counts.total) || - !Number.isFinite(summary.counts.passed) || - !Number.isFinite(summary.counts.failed) + !isNonNegativeInteger(summary.counts.total) || + !isNonNegativeInteger(summary.counts.passed) || + !isNonNegativeInteger(summary.counts.failed) ) { return "QA suite summary missing numeric counts"; } if (!summary.run || typeof summary.run !== "object" || Array.isArray(summary.run)) { return "QA suite summary missing run metadata"; } + const statusCounts = { failed: 0, passed: 0, skipped: 0 }; + for (const scenario of summary.scenarios) { + if (!scenario || typeof scenario !== "object" || Array.isArray(scenario)) { + return "QA suite summary scenario entries must be objects"; + } + if (scenario.status === "pass") { + statusCounts.passed += 1; + } else if (scenario.status === "fail") { + statusCounts.failed += 1; + } else if (scenario.status === "skip") { + statusCounts.skipped += 1; + } else { + return `QA suite summary scenario has invalid status: ${String(scenario.status)}`; + } + } + if (summary.counts.total !== summary.scenarios.length) { + return `QA suite summary total count mismatch: counts.total=${summary.counts.total}, scenarios=${summary.scenarios.length}`; + } + if (summary.counts.passed !== statusCounts.passed) { + return `QA suite summary passed count mismatch: counts.passed=${summary.counts.passed}, passed scenarios=${statusCounts.passed}`; + } + if (summary.counts.failed !== statusCounts.failed) { + return `QA suite summary failed count mismatch: counts.failed=${summary.counts.failed}, failed scenarios=${statusCounts.failed}`; + } + if ( + summary.counts.skipped !== undefined && + (!isNonNegativeInteger(summary.counts.skipped) || + summary.counts.skipped !== statusCounts.skipped) + ) { + return `QA suite summary skipped count mismatch: counts.skipped=${String(summary.counts.skipped)}, skipped scenarios=${statusCounts.skipped}`; + } return null; } +function isNonNegativeInteger(value) { + return Number.isInteger(value) && value >= 0; +} + export { collectQaBaselineRegressionObservations, collectGatewayCpuObservations, diff --git a/test/scripts/plugin-gateway-gauntlet.test.ts b/test/scripts/plugin-gateway-gauntlet.test.ts index 9d208a77eb07..87bf26478d43 100644 --- a/test/scripts/plugin-gateway-gauntlet.test.ts +++ b/test/scripts/plugin-gateway-gauntlet.test.ts @@ -1080,6 +1080,86 @@ setInterval(() => {}, 1000); await fs.rm(summary.isolatedRunRoot, { recursive: true, force: true }); }); + it("fails successful QA chunks whose scenario statuses disagree with counts", async () => { + const outputDir = path.join(repoRoot, "artifacts"); + const qaSummaryJson = JSON.stringify({ + counts: { failed: 0, passed: 1, total: 2 }, + metrics: { gatewayCpuCoreRatio: 0, wallMs: 1 }, + run: { + concurrency: 1, + fastMode: false, + finishedAt: "2026-05-30T00:00:01.000Z", + primaryModel: "mock-openai/gpt-5.5", + primaryModelName: "gpt-5.5", + primaryProvider: "mock-openai", + providerMode: "mock-openai", + scenarioIds: ["channel-chat-baseline", "gateway-restart-inflight-run"], + startedAt: "2026-05-30T00:00:00.000Z", + }, + scenarios: [ + { name: "channel-chat-baseline", status: "pass", steps: [] }, + { name: "gateway-restart-inflight-run", status: "fail", steps: [] }, + ], + }); + await writeManifest("alpha", "openclaw.plugin.json", JSON.stringify({ id: "alpha" })); + await fs.writeFile(path.join(repoRoot, "extensions", "alpha", "index.ts"), "export {};\n"); + await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true }); + await fs.writeFile( + path.join(repoRoot, "scripts", "run-node.mjs"), + [ + 'import fs from "node:fs";', + 'import path from "node:path";', + 'const outputArgIndex = process.argv.indexOf("--output-dir");', + "const outputDir = path.resolve(process.cwd(), process.argv[outputArgIndex + 1]);", + "fs.mkdirSync(outputDir, { recursive: true });", + `fs.writeFileSync(path.join(outputDir, "qa-suite-summary.json"), ${JSON.stringify(qaSummaryJson)}, "utf8");`, + ].join("\n"), + "utf8", + ); + + const result = spawnSync( + process.execPath, + [ + path.resolve("scripts/check-plugin-gateway-gauntlet.mjs"), + "--repo-root", + repoRoot, + "--output-dir", + outputDir, + "--skip-prebuild", + "--skip-lifecycle", + "--skip-slash-help", + "--plugin", + "alpha", + "--qa-scenario", + "channel-chat-baseline", + "--qa-scenario", + "gateway-restart-inflight-run", + ], + { + cwd: path.resolve("."), + encoding: "utf8", + }, + ); + + expect(result.status, result.stdout).toBe(1); + expect(result.stdout).toContain("diagnostic=qa-summary-invalid"); + const summary = JSON.parse( + await fs.readFile(path.join(outputDir, "plugin-gateway-gauntlet-summary.json"), "utf8"), + ); + expect(summary.failures).toEqual([ + expect.objectContaining({ + diagnosticDetail: + "QA suite summary failed count mismatch: counts.failed=0, failed scenarios=1", + diagnosticFailure: "qa-summary-invalid", + phase: "qa:rpc", + pluginId: "alpha", + status: 0, + }), + ]); + expect(summary.isolatedRunRootPreserved).toBe(true); + await fs.rm(summary.isolatedRunRoot, { recursive: true, force: true }); + }); + it("fails successful QA chunks that do not write the requested summary", async () => { const outputDir = path.join(repoRoot, "artifacts"); await writeManifest("alpha", "openclaw.plugin.json", JSON.stringify({ id: "alpha" }));