fix(ci): accept valid Kova PASS records without violations (#103002)

scripts/lib/kova-report-gate.mjs

@@ -31,6 +31,14 @@ function array(value, label) {
   check(Array.isArray(value), `invalid ${label}`);
   return value;
 }
+
+function recordViolations(record) {
+  if (!Object.hasOwn(record, "violations")) {
+    return [];
+  }
+  return array(record.violations, "record violations");
+}
+
 function count(value, label, { positive = false } = {}) {
   check(Number.isSafeInteger(value) && value >= (positive ? 1 : 0), `invalid ${label}`);
   return value;
@@ -428,7 +436,7 @@ function validateProfiledFailure(record, card, group) {
     group.resourceInterpretation === "instrumented",
     "failed record group was not instrumented",
   );
-  const violations = array(record.violations, "record violations");
+  const violations = recordViolations(record);
   check(violations.length > 0, "failed record had no violations");
   for (const violationValue of violations) {
     const violation = object(violationValue, "violation");
@@ -488,7 +496,7 @@ export function evaluateToleratedPartialKovaReport(report) {
       "PARTIAL report had a non-PASS record",
     );
     check(
-      records.every((record) => array(record.violations, "record violations").length === 0),
+      records.every((record) => recordViolations(record).length === 0),
       "PARTIAL report had violations",
     );
   });
@@ -505,7 +513,7 @@ export function evaluateToleratedProfiledKovaReport(report) {
     check(
       records
         .filter((record) => record.status === "PASS")
-        .every((record) => array(record.violations, "record violations").length === 0),
+        .every((record) => recordViolations(record).length === 0),
       "PASS record had violations",
     );
     const failed = records.filter((record) => record.status === "FAIL");

test/scripts/kova-report-gate.test.ts

@@ -15,6 +15,11 @@ type PathPart = number | string;
 type ReportMutation = [string, (report: JsonObject) => void];
 
 const tempRoots: string[] = [];
+const malformedViolationLists: Array<[string, unknown]> = [
+  ["null", null],
+  ["object", {}],
+  ["string", "none"],
+];
 const SCRIPT_PATH = "scripts/lib/kova-report-gate.mjs";
 const SCENARIO = "agent-cold-warm-message";
 const STATE = "mock-openai-provider";
@@ -207,7 +212,6 @@ function partialReport(): JsonObject {
         state: { id: STATE },
         status: "PASS",
         surface: SURFACE,
-        violations: [],
       },
     ],
     schemaVersion: "kova.report.v1",
@@ -360,6 +364,28 @@ function blockingCard(report: JsonObject): JsonObject {
   return objectAt(card);
 }
 
+function addProfiledPassRecord(report: JsonObject): JsonObject {
+  const scenario = "passing-agent-message";
+  const records = arrayAt(report.records);
+  const passRecord = objectAt(structuredClone(records[0]));
+  passRecord.scenario = scenario;
+  passRecord.status = "PASS";
+  delete passRecord.violations;
+  records.push(passRecord);
+
+  const performance = objectAt(report.performance);
+  const groups = arrayAt(performance.groups);
+  const passGroup = objectAt(structuredClone(groups[0]));
+  passGroup.key = `${scenario}|${SURFACE}|${STATE}`;
+  passGroup.scenario = scenario;
+  passGroup.statuses = { PASS: 1 };
+  groups.push(passGroup);
+  performance.groupCount = 2;
+  performance.profiledRunCount = 2;
+  report.summary = { statuses: { FAIL: 1, PASS: 1 }, total: 2 };
+  return passRecord;
+}
+
 function writeReport(report: unknown): string {
   const root = mkdtempSync(join(tmpdir(), "openclaw-kova-report-"));
   tempRoots.push(root);
@@ -383,7 +409,7 @@ afterEach(() => {
 });
 
 describe("scripts/lib/kova-report-gate.mjs", () => {
-  it("accepts an exact filtered PARTIAL execution report", () => {
+  it("accepts omitted violations on a filtered PARTIAL PASS record", () => {
     expect(evaluateToleratedPartialKovaReport(partialReport())).toEqual({ ok: true });
     expect(evaluateToleratedKovaReport(partialReport())).toEqual({
       classification: "filtered-partial",
@@ -454,26 +480,38 @@ describe("scripts/lib/kova-report-gate.mjs", () => {
     expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true });
   });
 
+  it("accepts omitted violations on a profiled PASS record", () => {
+    const report = profiledResourceReport();
+    addProfiledPassRecord(report);
+
+    expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true });
+  });
+
+  it.each(malformedViolationLists)(
+    "rejects present non-array %s violations on a PARTIAL PASS record",
+    (_label, violations) => {
+      const report = partialReport();
+      setAt(report, ["records", 0, "violations"], violations);
+
+      expectPartialRejection(report);
+    },
+  );
+
+  it.each(malformedViolationLists)(
+    "rejects present non-array %s violations on a profiled PASS record",
+    (_label, violations) => {
+      const report = profiledResourceReport();
+      const passRecord = addProfiledPassRecord(report);
+      passRecord.violations = violations;
+
+      expectProfiledRejection(report);
+    },
+  );
+
   it("rejects hidden violations on PASS records", () => {
     const report = profiledResourceReport();
-    const scenario = "passing-agent-message";
-    const records = arrayAt(report.records);
-    const passRecord = objectAt(structuredClone(records[0]));
-    passRecord.scenario = scenario;
-    passRecord.status = "PASS";
+    const passRecord = addProfiledPassRecord(report);
     passRecord.violations = [{ message: "hidden violation" }];
-    records.push(passRecord);
-
-    const performance = objectAt(report.performance);
-    const groups = arrayAt(performance.groups);
-    const passGroup = objectAt(structuredClone(groups[0]));
-    passGroup.key = `${scenario}|${SURFACE}|${STATE}`;
-    passGroup.scenario = scenario;
-    passGroup.statuses = { PASS: 1 };
-    groups.push(passGroup);
-    performance.groupCount = 2;
-    performance.profiledRunCount = 2;
-    report.summary = { statuses: { FAIL: 1, PASS: 1 }, total: 2 };
 
     expectProfiledRejection(report);
   });
@@ -618,6 +656,14 @@ describe("scripts/lib/kova-report-gate.mjs", () => {
       "rejects violations not bound to direct measurements",
       (report) => setAt(report, ["records", 0, "violations", 0, "actual"], 900),
     ],
+    [
+      "rejects failed records with omitted violations",
+      (report) => deleteAt(report, ["records", 0, "violations"]),
+    ],
+    [
+      "rejects failed records with an empty violations list",
+      (report) => setAt(report, ["records", 0, "violations"], []),
+    ],
     [
       "rejects violations without expectations",
       (report) => setAt(report, ["records", 0, "violations", 0, "expected"], ""),