diff --git a/scripts/bench-cli-startup.ts b/scripts/bench-cli-startup.ts index f47e90781390..a1332d2335c5 100644 --- a/scripts/bench-cli-startup.ts +++ b/scripts/bench-cli-startup.ts @@ -552,7 +552,12 @@ function collectExitSummary(samples: Sample[]): string { } function buildConfigFixture(commandCase: CommandCase): Record | null { - if (commandCase.id !== "configGetGatewayPort" && commandCase.id !== "gatewayHealthJson") { + if ( + commandCase.id !== "configGetGatewayPort" && + commandCase.id !== "gatewayHealthJson" && + commandCase.id !== "health" && + commandCase.id !== "healthJson" + ) { return null; } const port = parseGatewayPortEnv(process.env.OPENCLAW_GATEWAY_PORT); diff --git a/scripts/test-cli-startup-bench-budget.mjs b/scripts/test-cli-startup-bench-budget.mjs index 4c078907db2c..031de44869e8 100644 --- a/scripts/test-cli-startup-bench-budget.mjs +++ b/scripts/test-cli-startup-bench-budget.mjs @@ -35,6 +35,11 @@ if (process.argv.slice(2).includes("--help")) { " Fail if avg first-output time regresses more than this percent", " --max-rss-regression-pct Fail if avg RSS regresses more than this percent", " --skip-baseline Skip fixture regression checks and enforce case contracts only", + " --skip-response-budgets Skip response first-output and exit budget contracts", + "", + "Non-x64 runs skip fixture regression checks by default because the", + "checked-in startup fixture is a canonical x64 budget. Response contracts still run. Set", + "OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force them.", " --help Show this help text", "", "Example:", @@ -63,6 +68,7 @@ try { maxRssRegressionPct: readBudgetEnvNumber("OPENCLAW_STARTUP_BENCH_MAX_RSS_REGRESSION_PCT") ?? 20, skipBaseline: false, + skipResponseBudgets: false, }, [ stringFlag("--baseline", "baseline"), @@ -76,6 +82,7 @@ try { budgetFloatFlag("--max-first-output-regression-pct", "maxFirstOutputRegressionPct"), budgetFloatFlag("--max-rss-regression-pct", "maxRssRegressionPct"), booleanFlag("--skip-baseline", "skipBaseline"), + booleanFlag("--skip-response-budgets", "skipResponseBudgets"), ], ); } catch (error) { @@ -83,6 +90,18 @@ try { process.exit(1); } +const shouldAutoSkipNonCanonicalBaselineChecks = + process.arch !== "x64" && process.env.OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH !== "1"; +if (shouldAutoSkipNonCanonicalBaselineChecks && !opts.skipBaseline) { + console.warn( + `[test-cli-startup-bench-budget] skipping x64 startup fixture budgets on ${process.arch}; response contracts and sample output validation still ran. Set OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force fixture checks.`, + ); + opts = { + ...opts, + skipBaseline: true, + }; +} + function resolveCurrentReportPath() { if (opts.report) { return opts.report; @@ -137,14 +156,24 @@ const shouldRequireEveryBaselineCase = opts.preset === "all"; let failed = false; +for (const [id] of baselineCases) { + if (shouldRequireEveryBaselineCase && !currentCases.has(id)) { + console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`); + failed = true; + } +} + +for (const currentCase of currentCases.values()) { + if ((currentCase.samples ?? []).length === 0) { + console.error(`[test-cli-startup-bench-budget] ${currentCase.name} has no measured samples.`); + failed = true; + } +} + if (!opts.skipBaseline) { for (const [id, baselineCase] of baselineCases) { const currentCase = currentCases.get(id); if (!currentCase) { - if (shouldRequireEveryBaselineCase) { - console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`); - failed = true; - } continue; } @@ -236,36 +265,42 @@ for (const currentCase of currentCases.values()) { failed = true; } - const firstOutputBudgetMs = contract.firstOutputBudgetMs; - const firstOutputMax = currentCase.summary?.firstOutputMs?.max; - if (Number.isFinite(firstOutputBudgetMs)) { - if (!Number.isFinite(firstOutputMax)) { + if (!opts.skipResponseBudgets) { + const firstOutputBudgetMs = contract.firstOutputBudgetMs; + const firstOutputMax = currentCase.summary?.firstOutputMs?.max; + if (Number.isFinite(firstOutputBudgetMs)) { + if (!Number.isFinite(firstOutputMax)) { + console.error( + `[test-cli-startup-bench-budget] ${currentCase.name} produced no stdout/stderr before exit; response contract requires first output within ${formatMs( + firstOutputBudgetMs, + )}.`, + ); + failed = true; + } else if (firstOutputMax > firstOutputBudgetMs) { + console.error( + `[test-cli-startup-bench-budget] ${currentCase.name} first output ${formatMs( + firstOutputMax, + )} exceeded contract ${formatMs(firstOutputBudgetMs)}.`, + ); + failed = true; + } + } + + const exitBudgetMs = contract.exitBudgetMs; + const durationMax = currentCase.summary?.durationMs?.max; + if ( + Number.isFinite(exitBudgetMs) && + Number.isFinite(durationMax) && + durationMax > exitBudgetMs + ) { console.error( - `[test-cli-startup-bench-budget] ${currentCase.name} produced no stdout/stderr before exit; response contract requires first output within ${formatMs( - firstOutputBudgetMs, - )}.`, - ); - failed = true; - } else if (firstOutputMax > firstOutputBudgetMs) { - console.error( - `[test-cli-startup-bench-budget] ${currentCase.name} first output ${formatMs( - firstOutputMax, - )} exceeded contract ${formatMs(firstOutputBudgetMs)}.`, + `[test-cli-startup-bench-budget] ${currentCase.name} exit ${formatMs( + durationMax, + )} exceeded contract ${formatMs(exitBudgetMs)}.`, ); failed = true; } } - - const exitBudgetMs = contract.exitBudgetMs; - const durationMax = currentCase.summary?.durationMs?.max; - if (Number.isFinite(exitBudgetMs) && Number.isFinite(durationMax) && durationMax > exitBudgetMs) { - console.error( - `[test-cli-startup-bench-budget] ${currentCase.name} exit ${formatMs( - durationMax, - )} exceeded contract ${formatMs(exitBudgetMs)}.`, - ); - failed = true; - } } if (failed) { diff --git a/test/scripts/bench-cli-startup.test.ts b/test/scripts/bench-cli-startup.test.ts index e14d3c5de42f..57128b860b91 100644 --- a/test/scripts/bench-cli-startup.test.ts +++ b/test/scripts/bench-cli-startup.test.ts @@ -187,40 +187,41 @@ describe("bench-cli-startup", () => { }); it("writes a config fixture for config get benchmarks", () => { - expect( - withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () => - testing.buildConfigFixture({ - id: "configGetGatewayPort", - name: "config get gateway.port", - args: ["config", "get", "gateway.port"], - presets: ["real"], - }), - ), - ).toEqual({ + const expectedFixture = { gateway: { auth: { mode: "none" }, bind: "loopback", mode: "local", port: 32123, }, - }); - expect( - withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () => - testing.buildConfigFixture({ - id: "gatewayHealthJson", - name: "gateway health --json", - args: ["gateway", "health", "--json"], - presets: ["real"], - }), - ), - ).toEqual({ - gateway: { - auth: { mode: "none" }, - bind: "loopback", - mode: "local", - port: 32123, + }; + for (const commandCase of [ + { + id: "configGetGatewayPort", + name: "config get gateway.port", + args: ["config", "get", "gateway.port"], + presets: ["real"], }, - }); + { + id: "gatewayHealthJson", + name: "gateway health --json", + args: ["gateway", "health", "--json"], + presets: ["real"], + }, + { id: "health", name: "health", args: ["health"], presets: ["startup", "real"] }, + { + id: "healthJson", + name: "health --json", + args: ["health", "--json"], + presets: ["startup"], + }, + ]) { + expect( + withEnv({ OPENCLAW_GATEWAY_PORT: undefined }, () => + testing.buildConfigFixture(commandCase), + ), + ).toEqual(expectedFixture); + } }); it("parses config fixture gateway ports strictly from env", () => { diff --git a/test/scripts/cli-startup-bench-spawner.test.ts b/test/scripts/cli-startup-bench-spawner.test.ts index d1611e1244df..e5bc992835b3 100644 --- a/test/scripts/cli-startup-bench-spawner.test.ts +++ b/test/scripts/cli-startup-bench-spawner.test.ts @@ -41,7 +41,7 @@ describe("CLI startup benchmark script spawners", () => { const makeCase = (id: string, name: string) => ({ id, name, - samples: [], + samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }], summary: { durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, firstOutputMs: null, @@ -88,7 +88,14 @@ describe("CLI startup benchmark script spawners", () => { "--preset", "all", ], - { cwd: process.cwd(), stdio: "pipe" }, + { + cwd: process.cwd(), + env: { + ...process.env, + OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH: "1", + }, + stdio: "pipe", + }, ), ).toThrow(); } finally { @@ -96,6 +103,121 @@ describe("CLI startup benchmark script spawners", () => { } }); + it("skips x64 startup budgets on noncanonical architectures", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-arch-test-")); + try { + const archShimPath = path.join(tmpDir, "arch-shim.mjs"); + const baselinePath = path.join(tmpDir, "baseline.json"); + const reportPath = path.join(tmpDir, "current.json"); + const slowCase = { + id: "slow", + name: "slow", + contract: { + firstOutputBudgetMs: 20, + exitBudgetMs: 20, + }, + samples: [{ ms: 10, firstOutputMs: 10, maxRssMb: 100, exitCode: 0, signal: null }], + summary: { + durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, + firstOutputMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 }, + maxRssMb: { avg: 100, p50: 100, p95: 100, min: 100, max: 100 }, + }, + }; + fs.writeFileSync( + archShimPath, + 'Object.defineProperty(process, "arch", { value: "arm64" });\n', + ); + fs.writeFileSync( + baselinePath, + JSON.stringify({ + primary: { + cases: [ + { + ...slowCase, + summary: { + ...slowCase.summary, + durationMs: { avg: 1, p50: 1, p95: 1, min: 1, max: 1 }, + }, + }, + ], + }, + }), + ); + fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [slowCase] } })); + + const result = spawnSync( + process.execPath, + [ + "--import", + archShimPath, + "scripts/test-cli-startup-bench-budget.mjs", + "--baseline", + baselinePath, + "--report", + reportPath, + "--preset", + "all", + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toContain("skipping x64 startup fixture budgets on arm64"); + expect(result.stderr).not.toContain("exceeded"); + + const slowResponseCase = { + ...slowCase, + contract: { + firstOutputBudgetMs: 1, + exitBudgetMs: 1, + }, + }; + fs.writeFileSync( + baselinePath, + JSON.stringify({ primary: { cases: [slowResponseCase] } }), + ); + fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [slowResponseCase] } })); + const responseBudgetResult = spawnSync( + process.execPath, + [ + "--import", + archShimPath, + "scripts/test-cli-startup-bench-budget.mjs", + "--baseline", + baselinePath, + "--report", + reportPath, + "--preset", + "all", + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + expect(responseBudgetResult.status).toBe(1); + expect(responseBudgetResult.stderr).toContain("first output 10.0ms exceeded contract 1.0ms"); + + fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [] } })); + const missingCaseResult = spawnSync( + process.execPath, + [ + "--import", + archShimPath, + "scripts/test-cli-startup-bench-budget.mjs", + "--baseline", + baselinePath, + "--report", + reportPath, + "--preset", + "all", + ], + { cwd: process.cwd(), encoding: "utf8" }, + ); + expect(missingCaseResult.status).toBe(1); + expect(missingCaseResult.stderr).toContain("missing current case slow"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it("rejects malformed startup budget env vars before reading reports", () => { const result = spawnSync(process.execPath, ["scripts/test-cli-startup-bench-budget.mjs"], { cwd: process.cwd(),