fix(ci): require source performance artifacts

scripts/openclaw-performance-source-summary.mjs

@@ -60,22 +60,31 @@ function readJsonIfExists(filePath) {
   return JSON.parse(fs.readFileSync(filePath, "utf8"));
 }
 
+function readRequiredJson(filePath, label) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`[source-performance] missing required ${label}: ${filePath}`);
+  }
+  return JSON.parse(fs.readFileSync(filePath, "utf8"));
+}
+
+function finiteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
 function formatMs(value) {
-  return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)}ms` : "n/a";
+  return finiteNumber(value) ? `${value.toFixed(1)}ms` : "n/a";
 }
 
 function formatMb(value) {
-  return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)}MB` : "n/a";
+  return finiteNumber(value) ? `${value.toFixed(1)}MB` : "n/a";
 }
 
 function formatBytesAsMb(value) {
-  return typeof value === "number" && Number.isFinite(value)
-    ? formatMb(value / 1024 / 1024)
-    : "n/a";
+  return finiteNumber(value) ? formatMb(value / 1024 / 1024) : "n/a";
 }
 
 function formatRatio(value) {
-  return typeof value === "number" && Number.isFinite(value) ? value.toFixed(3) : "n/a";
+  return finiteNumber(value) ? value.toFixed(3) : "n/a";
 }
 
 function metric(stats, key = "p50") {
@@ -134,32 +143,147 @@ function table(headers, rows) {
   ];
 }
 
-function loadMockHelloSummaries(sourceDir) {
+function validateMockHelloSummary(summary, filePath) {
+  const counts = summary?.counts;
+  if (
+    !finiteNumber(counts?.total) ||
+    !finiteNumber(counts?.passed) ||
+    !finiteNumber(counts?.failed) ||
+    counts.total <= 0 ||
+    counts.failed !== 0 ||
+    counts.passed !== counts.total
+  ) {
+    throw new Error(`[source-performance] invalid mock hello summary counts: ${filePath}`);
+  }
+  const metrics = summary?.metrics;
+  const requiredMetrics = [
+    "wallMs",
+    "gatewayCpuCoreRatio",
+    "gatewayProcessRssStartBytes",
+    "gatewayProcessRssEndBytes",
+    "gatewayProcessRssDeltaBytes",
+  ];
+  const missingMetric = requiredMetrics.find((key) => !finiteNumber(metrics?.[key]));
+  if (missingMetric) {
+    throw new Error(`[source-performance] missing mock hello metric ${missingMetric}: ${filePath}`);
+  }
+}
+
+function loadMockHelloSummaries(sourceDir, { required = false } = {}) {
   const root = path.join(sourceDir, "mock-hello");
   if (!fs.existsSync(root)) {
+    if (required) {
+      throw new Error(`[source-performance] missing required mock hello directory: ${root}`);
+    }
     return [];
   }
-  return fs
+  const summaries = fs
     .readdirSync(root, { withFileTypes: true })
     .filter((entry) => entry.isDirectory())
     .map((entry) => ({
       id: entry.name,
-      summary: readJsonIfExists(path.join(root, entry.name, "qa-suite-summary.json")),
+      summaryPath: path.join(root, entry.name, "qa-suite-summary.json"),
+    }))
+    .filter((entry) => fs.existsSync(entry.summaryPath))
+    .map((entry) => ({
+      id: entry.id,
+      summary: JSON.parse(fs.readFileSync(entry.summaryPath, "utf8")),
+      summaryPath: entry.summaryPath,
     }))
-    .filter((entry) => entry.summary != null)
     .toSorted((a, b) => a.id.localeCompare(b.id));
+  if (required && summaries.length === 0) {
+    throw new Error(`[source-performance] missing required mock hello summaries: ${root}`);
+  }
+  if (required) {
+    for (const entry of summaries) {
+      validateMockHelloSummary(entry.summary, entry.summaryPath);
+    }
+  }
+  return summaries.map(({ id, summary }) => ({ id, summary }));
 }
 
-function loadSourceArtifacts(sourceDir) {
+function validateStartupArtifact(startup, filePath) {
+  if (!Array.isArray(startup?.results) || startup.results.length === 0) {
+    throw new Error(`[source-performance] missing gateway startup results: ${filePath}`);
+  }
+  for (const result of startup.results) {
+    if (
+      !finiteNumber(result?.summary?.readyzMs?.p50) ||
+      !finiteNumber(result?.summary?.maxRssMb?.p95) ||
+      !finiteNumber(result?.summary?.cpuCoreRatio?.p95)
+    ) {
+      throw new Error(
+        `[source-performance] incomplete gateway startup metrics for ${result?.id ?? "unknown"}: ${filePath}`,
+      );
+    }
+  }
+}
+
+function validateCliArtifact(cli, filePath) {
+  if (!Array.isArray(cli?.primary?.cases) || cli.primary.cases.length === 0) {
+    throw new Error(`[source-performance] missing CLI startup cases: ${filePath}`);
+  }
+  for (const entry of cli.primary.cases) {
+    if (
+      !finiteNumber(entry?.summary?.durationMs?.p50) ||
+      !finiteNumber(entry?.summary?.maxRssMb?.p95)
+    ) {
+      throw new Error(
+        `[source-performance] incomplete CLI startup metrics for ${entry?.id ?? "unknown"}: ${filePath}`,
+      );
+    }
+  }
+}
+
+function validateExtensionMemoryArtifact(extensionMemory, filePath) {
+  if (!Array.isArray(extensionMemory?.topByDeltaMb) || extensionMemory.topByDeltaMb.length === 0) {
+    throw new Error(`[source-performance] missing extension memory rows: ${filePath}`);
+  }
+  for (const entry of extensionMemory.topByDeltaMb) {
+    if (!finiteNumber(entry?.maxRssMb) || !finiteNumber(entry?.deltaFromBaselineMb)) {
+      throw new Error(
+        `[source-performance] incomplete extension memory metrics for ${entry?.dir ?? "unknown"}: ${filePath}`,
+      );
+    }
+  }
+}
+
+function validateGatewaySummaryArtifact(gatewaySummary, filePath) {
+  if (!Array.isArray(gatewaySummary?.observations)) {
+    throw new Error(`[source-performance] missing gateway observation summary: ${filePath}`);
+  }
+}
+
+function loadSourceArtifacts(sourceDir, { required = false } = {}) {
   if (!sourceDir || !fs.existsSync(sourceDir)) {
+    if (required) {
+      throw new Error(`[source-performance] missing required source dir: ${sourceDir}`);
+    }
     return null;
   }
-  return {
-    startup: readJsonIfExists(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json")),
-    cli: readJsonIfExists(path.join(sourceDir, "cli-startup.json")),
-    extensionMemory: readJsonIfExists(path.join(sourceDir, "extension-memory.json")),
-    mockHelloSummaries: loadMockHelloSummaries(sourceDir),
+  const stat = fs.statSync(sourceDir);
+  if (!stat.isDirectory()) {
+    throw new Error(`[source-performance] source path is not a directory: ${sourceDir}`);
+  }
+  const startupPath = path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json");
+  const cliPath = path.join(sourceDir, "cli-startup.json");
+  const extensionMemoryPath = path.join(sourceDir, "extension-memory.json");
+  const artifacts = {
+    startup: required
+      ? readRequiredJson(startupPath, "gateway startup artifact")
+      : readJsonIfExists(startupPath),
+    cli: required ? readRequiredJson(cliPath, "CLI startup artifact") : readJsonIfExists(cliPath),
+    extensionMemory: required
+      ? readRequiredJson(extensionMemoryPath, "extension memory artifact")
+      : readJsonIfExists(extensionMemoryPath),
+    mockHelloSummaries: loadMockHelloSummaries(sourceDir, { required }),
   };
+  if (required) {
+    validateStartupArtifact(artifacts.startup, startupPath);
+    validateCliArtifact(artifacts.cli, cliPath);
+    validateExtensionMemoryArtifact(artifacts.extensionMemory, extensionMemoryPath);
+  }
+  return artifacts;
 }
 
 function buildStartupRows(startup) {
@@ -350,15 +474,12 @@ function buildObservationRows(summary) {
   ]);
 }
 
-function buildMarkdown(sourceDir, baselineSourceDir) {
-  const current = loadSourceArtifacts(sourceDir) ?? {
-    startup: null,
-    cli: null,
-    extensionMemory: null,
-    mockHelloSummaries: [],
-  };
+export function buildMarkdown(sourceDir, baselineSourceDir) {
+  const current = loadSourceArtifacts(sourceDir, { required: true });
   const baseline = loadSourceArtifacts(baselineSourceDir);
-  const gatewaySummary = readJsonIfExists(path.join(sourceDir, "gateway-cpu", "summary.json"));
+  const gatewaySummaryPath = path.join(sourceDir, "gateway-cpu", "summary.json");
+  const gatewaySummary = readRequiredJson(gatewaySummaryPath, "gateway observation summary");
+  validateGatewaySummaryArtifact(gatewaySummary, gatewaySummaryPath);
   const memoryDeltaRows = buildMemoryDeltaRows(current, baseline);
 
   const lines = [

test/scripts/openclaw-performance-source-summary.test.ts

@@ -1,6 +1,85 @@
+import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
-import { describe, expect, it } from "vitest";
-import { parseArgs } from "../../scripts/openclaw-performance-source-summary.mjs";
+import { afterEach, describe, expect, it } from "vitest";
+import { buildMarkdown, parseArgs } from "../../scripts/openclaw-performance-source-summary.mjs";
+
+const tmpRoots: string[] = [];
+
+function mkTmpRoot() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-source-summary-"));
+  tmpRoots.push(root);
+  return root;
+}
+
+function writeJson(filePath: string, value: unknown) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(value), "utf8");
+}
+
+function writeSourceFixture(sourceDir: string) {
+  writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), {
+    results: [
+      {
+        id: "default",
+        name: "default",
+        summary: {
+          readyzMs: { p50: 12, p95: 18 },
+          healthzMs: { p50: 5 },
+          httpListenLogMs: { p50: 8 },
+          gatewayReadyLogMs: { p50: 9 },
+          firstOutputMs: { p50: 30 },
+          maxRssMb: { p95: 120 },
+          cpuCoreRatio: { p95: 0.25 },
+          startupTrace: {
+            "memory.ready.heapUsedMb": { p50: 30, p95: 32 },
+            "phase.load": { p50: 7, p95: 8 },
+          },
+        },
+      },
+    ],
+  });
+  writeJson(path.join(sourceDir, "gateway-cpu", "summary.json"), {
+    observations: [],
+  });
+  writeJson(path.join(sourceDir, "cli-startup.json"), {
+    primary: {
+      cases: [
+        {
+          id: "gatewayHealthJson",
+          name: "gateway health json",
+          summary: {
+            durationMs: { p50: 10, p95: 14 },
+            maxRssMb: { p95: 90 },
+            exitSummary: "code:0x3",
+          },
+        },
+      ],
+    },
+  });
+  writeJson(path.join(sourceDir, "extension-memory.json"), {
+    topByDeltaMb: [
+      { dir: "extensions/browser", maxRssMb: 80, deltaFromBaselineMb: 12, status: "ok" },
+    ],
+  });
+  writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {
+    counts: { failed: 0, passed: 1, total: 1 },
+    metrics: {
+      gatewayCpuCoreRatio: 0.15,
+      gatewayProcessRssDeltaBytes: 1024 * 1024,
+      gatewayProcessRssEndBytes: 91 * 1024 * 1024,
+      gatewayProcessRssStartBytes: 90 * 1024 * 1024,
+      wallMs: 250,
+    },
+    run: { primaryModel: "mock-openai/perf" },
+  });
+}
+
+afterEach(() => {
+  for (const root of tmpRoots.splice(0)) {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
 
 describe("parseArgs", () => {
   it("parses source summary paths", () => {
@@ -30,3 +109,49 @@ describe("parseArgs", () => {
     }
   });
 });
+
+describe("buildMarkdown", () => {
+  it("renders source performance fixtures with required artifacts", () => {
+    const sourceDir = mkTmpRoot();
+    writeSourceFixture(sourceDir);
+
+    expect(buildMarkdown(sourceDir, null)).toContain("run-001");
+    expect(buildMarkdown(sourceDir, null)).toContain("gateway health json");
+  });
+
+  it("rejects a missing source directory", () => {
+    expect(() => buildMarkdown(path.join(mkTmpRoot(), "missing"), null)).toThrow(
+      "[source-performance] missing required source dir:",
+    );
+  });
+
+  it("rejects missing source performance artifacts", () => {
+    const sourceDir = mkTmpRoot();
+
+    expect(() => buildMarkdown(sourceDir, null)).toThrow(
+      "[source-performance] missing required gateway startup artifact:",
+    );
+  });
+
+  it("rejects malformed mock hello summaries", () => {
+    const sourceDir = mkTmpRoot();
+    writeSourceFixture(sourceDir);
+    writeJson(path.join(sourceDir, "mock-hello", "run-001", "qa-suite-summary.json"), {});
+
+    expect(() => buildMarkdown(sourceDir, null)).toThrow(
+      "[source-performance] invalid mock hello summary counts:",
+    );
+  });
+
+  it("rejects gateway startup artifacts without resource metrics", () => {
+    const sourceDir = mkTmpRoot();
+    writeSourceFixture(sourceDir);
+    writeJson(path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"), {
+      results: [{ id: "default", summary: { readyzMs: { p50: 12 } } }],
+    });
+
+    expect(() => buildMarkdown(sourceDir, null)).toThrow(
+      "[source-performance] incomplete gateway startup metrics for default:",
+    );
+  });
+});