From a9578436dee47447213bbade63002f8f88666512 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 9 Jul 2026 11:05:32 -0700 Subject: [PATCH] fix(ci): accept valid Kova PASS records without violations (#103002) --- scripts/lib/kova-report-gate.mjs | 14 ++++- test/scripts/kova-report-gate.test.ts | 84 +++++++++++++++++++++------ 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/scripts/lib/kova-report-gate.mjs b/scripts/lib/kova-report-gate.mjs index 60c8f7a18588..742551c48d0a 100644 --- a/scripts/lib/kova-report-gate.mjs +++ b/scripts/lib/kova-report-gate.mjs @@ -31,6 +31,14 @@ function array(value, label) { check(Array.isArray(value), `invalid ${label}`); return value; } + +function recordViolations(record) { + if (!Object.hasOwn(record, "violations")) { + return []; + } + return array(record.violations, "record violations"); +} + function count(value, label, { positive = false } = {}) { check(Number.isSafeInteger(value) && value >= (positive ? 1 : 0), `invalid ${label}`); return value; @@ -428,7 +436,7 @@ function validateProfiledFailure(record, card, group) { group.resourceInterpretation === "instrumented", "failed record group was not instrumented", ); - const violations = array(record.violations, "record violations"); + const violations = recordViolations(record); check(violations.length > 0, "failed record had no violations"); for (const violationValue of violations) { const violation = object(violationValue, "violation"); @@ -488,7 +496,7 @@ export function evaluateToleratedPartialKovaReport(report) { "PARTIAL report had a non-PASS record", ); check( - records.every((record) => array(record.violations, "record violations").length === 0), + records.every((record) => recordViolations(record).length === 0), "PARTIAL report had violations", ); }); @@ -505,7 +513,7 @@ export function evaluateToleratedProfiledKovaReport(report) { check( records .filter((record) => record.status === "PASS") - .every((record) => array(record.violations, "record violations").length === 0), + .every((record) => recordViolations(record).length === 0), "PASS record had violations", ); const failed = records.filter((record) => record.status === "FAIL"); diff --git a/test/scripts/kova-report-gate.test.ts b/test/scripts/kova-report-gate.test.ts index a6f840a40532..775a4b38306b 100644 --- a/test/scripts/kova-report-gate.test.ts +++ b/test/scripts/kova-report-gate.test.ts @@ -15,6 +15,11 @@ type PathPart = number | string; type ReportMutation = [string, (report: JsonObject) => void]; const tempRoots: string[] = []; +const malformedViolationLists: Array<[string, unknown]> = [ + ["null", null], + ["object", {}], + ["string", "none"], +]; const SCRIPT_PATH = "scripts/lib/kova-report-gate.mjs"; const SCENARIO = "agent-cold-warm-message"; const STATE = "mock-openai-provider"; @@ -207,7 +212,6 @@ function partialReport(): JsonObject { state: { id: STATE }, status: "PASS", surface: SURFACE, - violations: [], }, ], schemaVersion: "kova.report.v1", @@ -360,6 +364,28 @@ function blockingCard(report: JsonObject): JsonObject { return objectAt(card); } +function addProfiledPassRecord(report: JsonObject): JsonObject { + const scenario = "passing-agent-message"; + const records = arrayAt(report.records); + const passRecord = objectAt(structuredClone(records[0])); + passRecord.scenario = scenario; + passRecord.status = "PASS"; + delete passRecord.violations; + records.push(passRecord); + + const performance = objectAt(report.performance); + const groups = arrayAt(performance.groups); + const passGroup = objectAt(structuredClone(groups[0])); + passGroup.key = `${scenario}|${SURFACE}|${STATE}`; + passGroup.scenario = scenario; + passGroup.statuses = { PASS: 1 }; + groups.push(passGroup); + performance.groupCount = 2; + performance.profiledRunCount = 2; + report.summary = { statuses: { FAIL: 1, PASS: 1 }, total: 2 }; + return passRecord; +} + function writeReport(report: unknown): string { const root = mkdtempSync(join(tmpdir(), "openclaw-kova-report-")); tempRoots.push(root); @@ -383,7 +409,7 @@ afterEach(() => { }); describe("scripts/lib/kova-report-gate.mjs", () => { - it("accepts an exact filtered PARTIAL execution report", () => { + it("accepts omitted violations on a filtered PARTIAL PASS record", () => { expect(evaluateToleratedPartialKovaReport(partialReport())).toEqual({ ok: true }); expect(evaluateToleratedKovaReport(partialReport())).toEqual({ classification: "filtered-partial", @@ -454,26 +480,38 @@ describe("scripts/lib/kova-report-gate.mjs", () => { expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true }); }); + it("accepts omitted violations on a profiled PASS record", () => { + const report = profiledResourceReport(); + addProfiledPassRecord(report); + + expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true }); + }); + + it.each(malformedViolationLists)( + "rejects present non-array %s violations on a PARTIAL PASS record", + (_label, violations) => { + const report = partialReport(); + setAt(report, ["records", 0, "violations"], violations); + + expectPartialRejection(report); + }, + ); + + it.each(malformedViolationLists)( + "rejects present non-array %s violations on a profiled PASS record", + (_label, violations) => { + const report = profiledResourceReport(); + const passRecord = addProfiledPassRecord(report); + passRecord.violations = violations; + + expectProfiledRejection(report); + }, + ); + it("rejects hidden violations on PASS records", () => { const report = profiledResourceReport(); - const scenario = "passing-agent-message"; - const records = arrayAt(report.records); - const passRecord = objectAt(structuredClone(records[0])); - passRecord.scenario = scenario; - passRecord.status = "PASS"; + const passRecord = addProfiledPassRecord(report); passRecord.violations = [{ message: "hidden violation" }]; - records.push(passRecord); - - const performance = objectAt(report.performance); - const groups = arrayAt(performance.groups); - const passGroup = objectAt(structuredClone(groups[0])); - passGroup.key = `${scenario}|${SURFACE}|${STATE}`; - passGroup.scenario = scenario; - passGroup.statuses = { PASS: 1 }; - groups.push(passGroup); - performance.groupCount = 2; - performance.profiledRunCount = 2; - report.summary = { statuses: { FAIL: 1, PASS: 1 }, total: 2 }; expectProfiledRejection(report); }); @@ -618,6 +656,14 @@ describe("scripts/lib/kova-report-gate.mjs", () => { "rejects violations not bound to direct measurements", (report) => setAt(report, ["records", 0, "violations", 0, "actual"], 900), ], + [ + "rejects failed records with omitted violations", + (report) => deleteAt(report, ["records", 0, "violations"]), + ], + [ + "rejects failed records with an empty violations list", + (report) => setAt(report, ["records", 0, "violations"], []), + ], [ "rejects violations without expectations", (report) => setAt(report, ["records", 0, "violations", 0, "expected"], ""),