diff --git a/scripts/openclaw-performance-source-summary.mjs b/scripts/openclaw-performance-source-summary.mjs index 173f74d12cb6..8439d70f9090 100644 --- a/scripts/openclaw-performance-source-summary.mjs +++ b/scripts/openclaw-performance-source-summary.mjs @@ -60,22 +60,31 @@ function readJsonIfExists(filePath) { return JSON.parse(fs.readFileSync(filePath, "utf8")); } +function readRequiredJson(filePath, label) { + if (!fs.existsSync(filePath)) { + throw new Error(`[source-performance] missing required ${label}: ${filePath}`); + } + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function finiteNumber(value) { + return typeof value === "number" && Number.isFinite(value); +} + function formatMs(value) { - return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)}ms` : "n/a"; + return finiteNumber(value) ? `${value.toFixed(1)}ms` : "n/a"; } function formatMb(value) { - return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)}MB` : "n/a"; + return finiteNumber(value) ? `${value.toFixed(1)}MB` : "n/a"; } function formatBytesAsMb(value) { - return typeof value === "number" && Number.isFinite(value) - ? formatMb(value / 1024 / 1024) - : "n/a"; + return finiteNumber(value) ? formatMb(value / 1024 / 1024) : "n/a"; } function formatRatio(value) { - return typeof value === "number" && Number.isFinite(value) ? value.toFixed(3) : "n/a"; + return finiteNumber(value) ? value.toFixed(3) : "n/a"; } function metric(stats, key = "p50") { @@ -134,32 +143,147 @@ function table(headers, rows) { ]; } -function loadMockHelloSummaries(sourceDir) { +function validateMockHelloSummary(summary, filePath) { + const counts = summary?.counts; + if ( + !finiteNumber(counts?.total) || + !finiteNumber(counts?.passed) || + !finiteNumber(counts?.failed) || + counts.total <= 0 || + counts.failed !== 0 || + counts.passed !== counts.total + ) { + throw new Error(`[source-performance] invalid mock hello summary counts: ${filePath}`); + } + const metrics = summary?.metrics; + const requiredMetrics = [ + "wallMs", + "gatewayCpuCoreRatio", + "gatewayProcessRssStartBytes", + "gatewayProcessRssEndBytes", + "gatewayProcessRssDeltaBytes", + ]; + const missingMetric = requiredMetrics.find((key) => !finiteNumber(metrics?.[key])); + if (missingMetric) { + throw new Error(`[source-performance] missing mock hello metric ${missingMetric}: ${filePath}`); + } +} + +function loadMockHelloSummaries(sourceDir, { required = false } = {}) { const root = path.join(sourceDir, "mock-hello"); if (!fs.existsSync(root)) { + if (required) { + throw new Error(`[source-performance] missing required mock hello directory: ${root}`); + } return []; } - return fs + const summaries = fs .readdirSync(root, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) .map((entry) => ({ id: entry.name, - summary: readJsonIfExists(path.join(root, entry.name, "qa-suite-summary.json")), + summaryPath: path.join(root, entry.name, "qa-suite-summary.json"), + })) + .filter((entry) => fs.existsSync(entry.summaryPath)) + .map((entry) => ({ + id: entry.id, + summary: JSON.parse(fs.readFileSync(entry.summaryPath, "utf8")), + summaryPath: entry.summaryPath, })) - .filter((entry) => entry.summary != null) .toSorted((a, b) => a.id.localeCompare(b.id)); + if (required && summaries.length === 0) { + throw new Error(`[source-performance] missing required mock hello summaries: ${root}`); + } + if (required) { + for (const entry of summaries) { + validateMockHelloSummary(entry.summary, entry.summaryPath); + } + } + return summaries.map(({ id, summary }) => ({ id, summary })); } -function loadSourceArtifacts(sourceDir) { +function validateStartupArtifact(startup, filePath) { + if (!Array.isArray(startup?.results) || startup.results.length === 0) { + throw new Error(`[source-performance] missing gateway startup results: ${filePath}`); + } + for (const result of startup.results) { + if ( + !finiteNumber(result?.summary?.readyzMs?.p50) || + !finiteNumber(result?.summary?.maxRssMb?.p95) || + !finiteNumber(result?.summary?.cpuCoreRatio?.p95) + ) { + throw new Error( + `[source-performance] incomplete gateway startup metrics for ${result?.id ?? "unknown"}: ${filePath}`, + ); + } + } +} + +function validateCliArtifact(cli, filePath) { + if (!Array.isArray(cli?.primary?.cases) || cli.primary.cases.length === 0) { + throw new Error(`[source-performance] missing CLI startup cases: ${filePath}`); + } + for (const entry of cli.primary.cases) { + if ( + !finiteNumber(entry?.summary?.durationMs?.p50) || + !finiteNumber(entry?.summary?.maxRssMb?.p95) + ) { + throw new Error( + `[source-performance] incomplete CLI startup metrics for ${entry?.id ?? "unknown"}: ${filePath}`, + ); + } + } +} + +function validateExtensionMemoryArtifact(extensionMemory, filePath) { + if (!Array.isArray(extensionMemory?.topByDeltaMb) || extensionMemory.topByDeltaMb.length === 0) { + throw new Error(`[source-performance] missing extension memory rows: ${filePath}`); + } + for (const entry of extensionMemory.topByDeltaMb) { + if (!finiteNumber(entry?.maxRssMb) || !finiteNumber(entry?.deltaFromBaselineMb)) { + throw new Error( + `[source-performance] incomplete extension memory metrics for ${entry?.dir ?? "unknown"}: ${filePath}`, + ); + } + } +} + +function validateGatewaySummaryArtifact(gatewaySummary, filePath) { + if (!Array.isArray(gatewaySummary?.observations)) { + throw new Error(`[source-performance] missing gateway observation summary: ${filePath}`); + } +} + +function loadSourceArtifacts(sourceDir, { required = false } = {}) { if (!sourceDir || !fs.existsSync(sourceDir)) { + if (required) { + throw new Error(`[source-performance] missing required source dir: ${sourceDir}`); + } return null; } - return { - startup: readJsonIfExists(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json")), - cli: readJsonIfExists(path.join(sourceDir, "cli-startup.json")), - extensionMemory: readJsonIfExists(path.join(sourceDir, "extension-memory.json")), - mockHelloSummaries: loadMockHelloSummaries(sourceDir), + const stat = fs.statSync(sourceDir); + if (!stat.isDirectory()) { + throw new Error(`[source-performance] source path is not a directory: ${sourceDir}`); + } + const startupPath = path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"); + const cliPath = path.join(sourceDir, "cli-startup.json"); + const extensionMemoryPath = path.join(sourceDir, "extension-memory.json"); + const artifacts = { + startup: required + ? readRequiredJson(startupPath, "gateway startup artifact") + : readJsonIfExists(startupPath), + cli: required ? readRequiredJson(cliPath, "CLI startup artifact") : readJsonIfExists(cliPath), + extensionMemory: required + ? readRequiredJson(extensionMemoryPath, "extension memory artifact") + : readJsonIfExists(extensionMemoryPath), + mockHelloSummaries: loadMockHelloSummaries(sourceDir, { required }), }; + if (required) { + validateStartupArtifact(artifacts.startup, startupPath); + validateCliArtifact(artifacts.cli, cliPath); + validateExtensionMemoryArtifact(artifacts.extensionMemory, extensionMemoryPath); + } + return artifacts; } function buildStartupRows(startup) { @@ -350,15 +474,12 @@ function buildObservationRows(summary) { ]); } -function buildMarkdown(sourceDir, baselineSourceDir) { - const current = loadSourceArtifacts(sourceDir) ?? { - startup: null, - cli: null, - extensionMemory: null, - mockHelloSummaries: [], - }; +export function buildMarkdown(sourceDir, baselineSourceDir) { + const current = loadSourceArtifacts(sourceDir, { required: true }); const baseline = loadSourceArtifacts(baselineSourceDir); - const gatewaySummary = readJsonIfExists(path.join(sourceDir, "gateway-cpu", "summary.json")); + const gatewaySummaryPath = path.join(sourceDir, "gateway-cpu", "summary.json"); + const gatewaySummary = readRequiredJson(gatewaySummaryPath, "gateway observation summary"); + validateGatewaySummaryArtifact(gatewaySummary, gatewaySummaryPath); const memoryDeltaRows = buildMemoryDeltaRows(current, baseline); const lines = [ diff --git a/test/scripts/openclaw-performance-source-summary.test.ts b/test/scripts/openclaw-performance-source-summary.test.ts index d37cd804448c..7cbf64a6ec0f 100644 --- a/test/scripts/openclaw-performance-source-summary.test.ts +++ b/test/scripts/openclaw-performance-source-summary.test.ts @@ -1,6 +1,85 @@ +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { parseArgs } from "../../scripts/openclaw-performance-source-summary.mjs"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildMarkdown, parseArgs } from "../../scripts/openclaw-performance-source-summary.mjs"; + +const tmpRoots: string[] = []; + +function mkTmpRoot() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-source-summary-")); + tmpRoots.push(root); + return root; +} + +function writeJson(filePath: string, value: unknown) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(value), "utf8"); +} + +function writeSourceFixture(sourceDir: string) { + writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), { + results: [ + { + id: "default", + name: "default", + summary: { + readyzMs: { p50: 12, p95: 18 }, + healthzMs: { p50: 5 }, + httpListenLogMs: { p50: 8 }, + gatewayReadyLogMs: { p50: 9 }, + firstOutputMs: { p50: 30 }, + maxRssMb: { p95: 120 }, + cpuCoreRatio: { p95: 0.25 }, + startupTrace: { + "memory.ready.heapUsedMb": { p50: 30, p95: 32 }, + "phase.load": { p50: 7, p95: 8 }, + }, + }, + }, + ], + }); + writeJson(path.join(sourceDir, "gateway-cpu", "summary.json"), { + observations: [], + }); + writeJson(path.join(sourceDir, "cli-startup.json"), { + primary: { + cases: [ + { + id: "gatewayHealthJson", + name: "gateway health json", + summary: { + durationMs: { p50: 10, p95: 14 }, + maxRssMb: { p95: 90 }, + exitSummary: "code:0x3", + }, + }, + ], + }, + }); + writeJson(path.join(sourceDir, "extension-memory.json"), { + topByDeltaMb: [ + { dir: "extensions/browser", maxRssMb: 80, deltaFromBaselineMb: 12, status: "ok" }, + ], + }); + writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), { + counts: { failed: 0, passed: 1, total: 1 }, + metrics: { + gatewayCpuCoreRatio: 0.15, + gatewayProcessRssDeltaBytes: 1024 * 1024, + gatewayProcessRssEndBytes: 91 * 1024 * 1024, + gatewayProcessRssStartBytes: 90 * 1024 * 1024, + wallMs: 250, + }, + run: { primaryModel: "mock-openai/perf" }, + }); +} + +afterEach(() => { + for (const root of tmpRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); describe("parseArgs", () => { it("parses source summary paths", () => { @@ -30,3 +109,49 @@ describe("parseArgs", () => { } }); }); + +describe("buildMarkdown", () => { + it("renders source performance fixtures with required artifacts", () => { + const sourceDir = mkTmpRoot(); + writeSourceFixture(sourceDir); + + expect(buildMarkdown(sourceDir, null)).toContain("run-001"); + expect(buildMarkdown(sourceDir, null)).toContain("gateway health json"); + }); + + it("rejects a missing source directory", () => { + expect(() => buildMarkdown(path.join(mkTmpRoot(), "missing"), null)).toThrow( + "[source-performance] missing required source dir:", + ); + }); + + it("rejects missing source performance artifacts", () => { + const sourceDir = mkTmpRoot(); + + expect(() => buildMarkdown(sourceDir, null)).toThrow( + "[source-performance] missing required gateway startup artifact:", + ); + }); + + it("rejects malformed mock hello summaries", () => { + const sourceDir = mkTmpRoot(); + writeSourceFixture(sourceDir); + writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {}); + + expect(() => buildMarkdown(sourceDir, null)).toThrow( + "[source-performance] invalid mock hello summary counts:", + ); + }); + + it("rejects gateway startup artifacts without resource metrics", () => { + const sourceDir = mkTmpRoot(); + writeSourceFixture(sourceDir); + writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), { + results: [{ id: "default", summary: { readyzMs: { p50: 12 } } }], + }); + + expect(() => buildMarkdown(sourceDir, null)).toThrow( + "[source-performance] incomplete gateway startup metrics for default:", + ); + }); +});