fix(scripts): bound gauntlet QA summaries

This commit is contained in:
Vincent Koc
2026-06-16 06:59:15 +02:00
parent 484ee14273
commit 96404a7bd5
2 changed files with 120 additions and 1 deletions

View File

@@ -6,9 +6,17 @@ import {
NON_PACKAGED_BUNDLED_PLUGIN_DIRS,
collectBundledPluginBuildEntries,
} from "./bundled-plugin-build-entries.mjs";
import { parsePositiveInt } from "./numeric-options.mjs";
const MANIFEST_NAMES = ["openclaw.plugin.json", "openclaw.plugin.json5"];
const ANSI_PATTERN = new RegExp(String.raw`\u001B\[[0-9;]*m`, "gu");
const QA_SUMMARY_MAX_BYTES_ENV = "OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_QA_SUMMARY_MAX_BYTES";
const DEFAULT_QA_SUMMARY_MAX_BYTES = 2 * 1024 * 1024;
function readPositiveIntEnv(name, fallback) {
const raw = process.env[name];
return raw === undefined || raw === "" ? fallback : parsePositiveInt(raw, name);
}
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
@@ -452,7 +460,7 @@ function readQaSuiteSummary(summaryPath) {
};
}
try {
const summary = JSON.parse(fs.readFileSync(summaryPath, "utf8"));
const summary = JSON.parse(readQaSuiteSummaryText(summaryPath));
const invalidReason = validateQaSuiteSummary(summary);
if (invalidReason) {
return {
@@ -482,6 +490,25 @@ function readQaSuiteSummary(summaryPath) {
}
}
function readQaSuiteSummaryText(summaryPath) {
const maxBytes = readPositiveIntEnv(QA_SUMMARY_MAX_BYTES_ENV, DEFAULT_QA_SUMMARY_MAX_BYTES);
const stat = fs.statSync(summaryPath);
if (!stat.isFile()) {
throw new Error(`QA suite summary is not a file: ${summaryPath}`);
}
if (stat.size > maxBytes) {
throw new Error(
`QA suite summary exceeded ${maxBytes} bytes: ${summaryPath} (${stat.size} bytes)`,
);
}
const text = fs.readFileSync(summaryPath, "utf8");
const bytes = Buffer.byteLength(text, "utf8");
if (bytes > maxBytes) {
throw new Error(`QA suite summary exceeded ${maxBytes} bytes: ${summaryPath} (${bytes} bytes)`);
}
return text;
}
function validateQaSuiteSummary(summary) {
if (!summary || typeof summary !== "object" || Array.isArray(summary)) {
return "QA suite summary must be a JSON object";

View File

@@ -846,6 +846,36 @@ setInterval(() => {}, 1000);
await expect(fs.stat(summary.isolatedRunRoot)).rejects.toHaveProperty("code", "ENOENT");
});
it("does not parse QA summary limit env when QA is skipped", () => {
const outputDir = path.join(repoRoot, "artifacts");
const result = spawnSync(
process.execPath,
[
path.resolve("scripts/check-plugin-gateway-gauntlet.mjs"),
"--repo-root",
repoRoot,
"--output-dir",
outputDir,
"--skip-prebuild",
"--skip-lifecycle",
"--skip-slash-help",
"--skip-qa",
"--allow-empty",
],
{
cwd: path.resolve("."),
encoding: "utf8",
env: {
...process.env,
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_QA_SUMMARY_MAX_BYTES: "not-a-number",
},
},
);
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain("failures=0");
});
it("probes plugin-owned slash help while the plugin is installed", async () => {
const outputDir = path.join(repoRoot, "artifacts");
await writeManifest(
@@ -1418,4 +1448,66 @@ setInterval(() => {}, 1000);
expect(summary.isolatedRunRootPreserved).toBe(true);
await fs.rm(summary.isolatedRunRoot, { recursive: true, force: true });
});
it("fails successful QA chunks that write oversized summary JSON", async () => {
const outputDir = path.join(repoRoot, "artifacts");
await writeManifest("alpha", "openclaw.plugin.json", JSON.stringify({ id: "alpha" }));
await fs.writeFile(path.join(repoRoot, "extensions", "alpha", "index.ts"), "export {};\n");
await fs.mkdir(path.join(repoRoot, "scripts"), { recursive: true });
await fs.writeFile(
path.join(repoRoot, "scripts", "run-node.mjs"),
[
'import fs from "node:fs";',
'import path from "node:path";',
'const outputArgIndex = process.argv.indexOf("--output-dir");',
"const outputDir = path.resolve(process.cwd(), process.argv[outputArgIndex + 1]);",
"fs.mkdirSync(outputDir, { recursive: true });",
'fs.writeFileSync(path.join(outputDir, "qa-suite-summary.json"), JSON.stringify({ filler: "x".repeat(128) }), "utf8");',
].join("\n"),
"utf8",
);
const result = spawnSync(
process.execPath,
[
path.resolve("scripts/check-plugin-gateway-gauntlet.mjs"),
"--repo-root",
repoRoot,
"--output-dir",
outputDir,
"--skip-prebuild",
"--skip-lifecycle",
"--skip-slash-help",
"--plugin",
"alpha",
"--qa-scenario",
"channel-chat-baseline",
],
{
cwd: path.resolve("."),
encoding: "utf8",
env: {
...process.env,
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_QA_SUMMARY_MAX_BYTES: "64",
},
},
);
expect(result.status, result.stdout).toBe(1);
expect(result.stdout).toContain("diagnostic=qa-summary-invalid");
const summary = JSON.parse(
await fs.readFile(path.join(outputDir, "plugin-gateway-gauntlet-summary.json"), "utf8"),
);
expect(summary.failures).toEqual([
expect.objectContaining({
diagnosticDetail: expect.stringContaining("QA suite summary exceeded 64 bytes"),
diagnosticFailure: "qa-summary-invalid",
phase: "qa:rpc",
pluginId: "alpha",
status: 0,
}),
]);
expect(summary.isolatedRunRootPreserved).toBe(true);
await fs.rm(summary.isolatedRunRoot, { recursive: true, force: true });
});
});