diff --git a/scripts/validate-qa-runtime-pair-summary.mjs b/scripts/validate-qa-runtime-pair-summary.mjs index f8c34519b140..74b9a32babcf 100644 --- a/scripts/validate-qa-runtime-pair-summary.mjs +++ b/scripts/validate-qa-runtime-pair-summary.mjs @@ -14,51 +14,50 @@ const HARD_RUNTIME_ERROR_CLASSES = new Set([ "capture-missing", ]); const EXPLICIT_CODEX_GAP_PREFIXES = ["known-harness-gap ", "codex-native-workspace "]; -const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([ - [ - "311047822ecdde24e824d839ab105ef08f17be00:core", - { - scenarioIds: [ - "instruction-followthrough-repo-contract", - "subagent-fanout-synthesis", - "subagent-handoff", - "subagent-stale-child-links", - "config-restart-capability-flip", - "image-understanding-attachment", - "memory-recall", - "thread-memory-isolation", - "model-switch-tool-continuity", - "approval-turn-tool-followthrough", - "codex-plugin-pinned-new", - "codex-plugin-pinned-old", - "compaction-retry-mutating-tool", - "runtime-first-hour-20-turn", - "runtime-tool-apply-patch", - "runtime-tool-bash", - "runtime-tool-edit", - "runtime-tool-exec", - "runtime-tool-fs-list", - "runtime-tool-fs-read", - "runtime-tool-fs-write", - "runtime-tool-grep", - "runtime-tool-session-status", - "runtime-tool-sessions-spawn", - "runtime-tool-web-fetch", - "runtime-tool-web-search", - "source-docs-discovery-report", - ], - gapScenarioIds: [ - "runtime-tool-apply-patch", - "runtime-tool-bash", - "runtime-tool-edit", - "runtime-tool-exec", - "runtime-tool-fs-list", - "runtime-tool-fs-read", - "runtime-tool-fs-write", - "runtime-tool-grep", - ], - }, +const FROZEN_CORE_RUNTIME_PAIR_MANIFEST = { + scenarioIds: [ + "instruction-followthrough-repo-contract", + "subagent-fanout-synthesis", + "subagent-handoff", + "subagent-stale-child-links", + "config-restart-capability-flip", + "image-understanding-attachment", + "memory-recall", + "thread-memory-isolation", + "model-switch-tool-continuity", + "approval-turn-tool-followthrough", + "codex-plugin-pinned-new", + "codex-plugin-pinned-old", + "compaction-retry-mutating-tool", + "runtime-first-hour-20-turn", + "runtime-tool-apply-patch", + "runtime-tool-bash", + "runtime-tool-edit", + "runtime-tool-exec", + "runtime-tool-fs-list", + "runtime-tool-fs-read", + "runtime-tool-fs-write", + "runtime-tool-grep", + "runtime-tool-session-status", + "runtime-tool-sessions-spawn", + "runtime-tool-web-fetch", + "runtime-tool-web-search", + "source-docs-discovery-report", ], + gapScenarioIds: [ + "runtime-tool-apply-patch", + "runtime-tool-bash", + "runtime-tool-edit", + "runtime-tool-exec", + "runtime-tool-fs-list", + "runtime-tool-fs-read", + "runtime-tool-fs-write", + "runtime-tool-grep", + ], +}; +const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([ + ["311047822ecdde24e824d839ab105ef08f17be00:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], + ["c37af96b18776fecc9e24268f27fc89b563481bf:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], ]); function isRecord(value) { diff --git a/test/scripts/validate-qa-runtime-pair-summary.test.ts b/test/scripts/validate-qa-runtime-pair-summary.test.ts index 01ca4cf1cf11..27dd95c3a988 100644 --- a/test/scripts/validate-qa-runtime-pair-summary.test.ts +++ b/test/scripts/validate-qa-runtime-pair-summary.test.ts @@ -224,14 +224,32 @@ describe("frozen QA runtime-pair summary validation", () => { ); }); + it.each(["311047822ecdde24e824d839ab105ef08f17be00", "c37af96b18776fecc9e24268f27fc89b563481bf"])( + "accepts the exact frozen core manifest for %s", + (targetSha) => { + const fixture = frozenCoreSummary(); + expect( + validateQaRuntimePairSummary(fixture, { + requireExplicitGap: true, + targetSha, + lane: "core", + }), + ).toMatchObject({ skipped: 8 }); + }, + ); + + it("rejects an arbitrary target with otherwise identical frozen evidence", () => { + expect(() => + validateQaRuntimePairSummary(frozenCoreSummary(), { + requireExplicitGap: true, + targetSha: "0000000000000000000000000000000000000000", + lane: "core", + }), + ).toThrow("trusted frozen-lane manifest"); + }); + it("pins the exact frozen scenarios that may use the explicit gap exception", () => { - const options = { - requireExplicitGap: true, - targetSha: "311047822ecdde24e824d839ab105ef08f17be00", - lane: "core", - }; const fixture = frozenCoreSummary(); - expect(validateQaRuntimePairSummary(fixture, options)).toMatchObject({ skipped: 8 }); const expectedGap = fixture.scenarios.find( (entry) => entry.runtimeParity.scenarioId === "runtime-tool-apply-patch", @@ -248,9 +266,13 @@ describe("frozen QA runtime-pair summary validation", () => { unrelated.runtimeParity.cells.codex.status = "skip"; unrelated.runtimeParity.cells.codex.details = "known-harness-gap exec: tracked"; - expect(() => validateQaRuntimePairSummary(fixture, options)).toThrow( - "trusted frozen-lane manifest", - ); + expect(() => + validateQaRuntimePairSummary(fixture, { + requireExplicitGap: true, + targetSha: "c37af96b18776fecc9e24268f27fc89b563481bf", + lane: "core", + }), + ).toThrow("trusted frozen-lane manifest"); }); it("cross-checks generated report JSON and Markdown", () => {