dilim/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-13 17:07:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(release): trust corrected beta runtime evidence (#114838)

Browse Source 
* fix(release): trust corrected beta runtime evidence

* style(test): format runtime evidence cases
This commit is contained in:
Peter Steinberger
2026-07-27 21:14:41 -04:00
committed by GitHub
parent d5907ca619
commit 5ae4a8fb8e
2 changed files with 74 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scripts/validate-qa-runtime-pair-summary.mjs
View File

@@ -14,51 +14,50 @@ const HARD_RUNTIME_ERROR_CLASSES = new Set([
"capture-missing",
]);
const EXPLICIT_CODEX_GAP_PREFIXES = ["known-harness-gap ", "codex-native-workspace "];
const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([
[
"311047822ecdde24e824d839ab105ef08f17be00:core",
{
scenarioIds: [
"instruction-followthrough-repo-contract",
"subagent-fanout-synthesis",
"subagent-handoff",
"subagent-stale-child-links",
"config-restart-capability-flip",
"image-understanding-attachment",
"memory-recall",
"thread-memory-isolation",
"model-switch-tool-continuity",
"approval-turn-tool-followthrough",
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"compaction-retry-mutating-tool",
"runtime-first-hour-20-turn",
"runtime-tool-apply-patch",
"runtime-tool-bash",
"runtime-tool-edit",
"runtime-tool-exec",
"runtime-tool-fs-list",
"runtime-tool-fs-read",
"runtime-tool-fs-write",
"runtime-tool-grep",
"runtime-tool-session-status",
"runtime-tool-sessions-spawn",
"runtime-tool-web-fetch",
"runtime-tool-web-search",
"source-docs-discovery-report",
],
gapScenarioIds: [
"runtime-tool-apply-patch",
"runtime-tool-bash",
"runtime-tool-edit",
"runtime-tool-exec",
"runtime-tool-fs-list",
"runtime-tool-fs-read",
"runtime-tool-fs-write",
"runtime-tool-grep",
],
},
const FROZEN_CORE_RUNTIME_PAIR_MANIFEST = {
scenarioIds: [
"instruction-followthrough-repo-contract",
"subagent-fanout-synthesis",
"subagent-handoff",
"subagent-stale-child-links",
"config-restart-capability-flip",
"image-understanding-attachment",
"memory-recall",
"thread-memory-isolation",
"model-switch-tool-continuity",
"approval-turn-tool-followthrough",
"codex-plugin-pinned-new",
"codex-plugin-pinned-old",
"compaction-retry-mutating-tool",
"runtime-first-hour-20-turn",
"runtime-tool-apply-patch",
"runtime-tool-bash",
"runtime-tool-edit",
"runtime-tool-exec",
"runtime-tool-fs-list",
"runtime-tool-fs-read",
"runtime-tool-fs-write",
"runtime-tool-grep",
"runtime-tool-session-status",
"runtime-tool-sessions-spawn",
"runtime-tool-web-fetch",
"runtime-tool-web-search",
"source-docs-discovery-report",
],
gapScenarioIds: [
"runtime-tool-apply-patch",
"runtime-tool-bash",
"runtime-tool-edit",
"runtime-tool-exec",
"runtime-tool-fs-list",
"runtime-tool-fs-read",
"runtime-tool-fs-write",
"runtime-tool-grep",
],
};
const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([
["311047822ecdde24e824d839ab105ef08f17be00:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST],
["c37af96b18776fecc9e24268f27fc89b563481bf:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST],
]);
function isRecord(value) {

40
test/scripts/validate-qa-runtime-pair-summary.test.ts
View File

@@ -224,14 +224,32 @@ describe("frozen QA runtime-pair summary validation", () => {
);
});
it.each(["311047822ecdde24e824d839ab105ef08f17be00", "c37af96b18776fecc9e24268f27fc89b563481bf"])(
"accepts the exact frozen core manifest for %s",
(targetSha) => {
const fixture = frozenCoreSummary();
expect(
validateQaRuntimePairSummary(fixture, {
requireExplicitGap: true,
targetSha,
lane: "core",
}),
).toMatchObject({ skipped: 8 });
},
);
it("rejects an arbitrary target with otherwise identical frozen evidence", () => {
expect(() =>
validateQaRuntimePairSummary(frozenCoreSummary(), {
requireExplicitGap: true,
targetSha: "0000000000000000000000000000000000000000",
lane: "core",
}),
).toThrow("trusted frozen-lane manifest");
});
it("pins the exact frozen scenarios that may use the explicit gap exception", () => {
const options = {
requireExplicitGap: true,
targetSha: "311047822ecdde24e824d839ab105ef08f17be00",
lane: "core",
};
const fixture = frozenCoreSummary();
expect(validateQaRuntimePairSummary(fixture, options)).toMatchObject({ skipped: 8 });
const expectedGap = fixture.scenarios.find(
(entry) => entry.runtimeParity.scenarioId === "runtime-tool-apply-patch",
@@ -248,9 +266,13 @@ describe("frozen QA runtime-pair summary validation", () => {
unrelated.runtimeParity.cells.codex.status = "skip";
unrelated.runtimeParity.cells.codex.details = "known-harness-gap exec: tracked";
expect(() => validateQaRuntimePairSummary(fixture, options)).toThrow(
"trusted frozen-lane manifest",
);
expect(() =>
validateQaRuntimePairSummary(fixture, {
requireExplicitGap: true,
targetSha: "c37af96b18776fecc9e24268f27fc89b563481bf",
lane: "core",
}),
).toThrow("trusted frozen-lane manifest");
});
it("cross-checks generated report JSON and Markdown", () => {