From 0004cfd59e3bf864803e0dee310de182cbde0dae Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 21 Jun 2026 07:44:49 +0200 Subject: [PATCH] test(scripts): route prompt snapshot helper changes --- scripts/check-changed.mjs | 18 +++++++ scripts/sync-codex-model-prompt-fixture.ts | 4 +- scripts/test-projects.test-support.mjs | 19 ++++++- test/scripts/changed-lanes.test.ts | 41 ++++++++++++++ test/scripts/prompt-snapshots.test.ts | 62 ++++++++++++++++++++++ test/scripts/test-projects.test.ts | 16 ++++++ 6 files changed, 157 insertions(+), 3 deletions(-) diff --git a/scripts/check-changed.mjs b/scripts/check-changed.mjs index 92908353cf59..15c58f988440 100644 --- a/scripts/check-changed.mjs +++ b/scripts/check-changed.mjs @@ -40,6 +40,10 @@ const LIVE_DOCKER_AUTH_SHELL_TARGETS = [ ]; const SHRINKWRAP_POLICY_PATH_RE = /^(?:npm-shrinkwrap\.json|package\.json|pnpm-lock\.yaml|pnpm-workspace\.yaml|scripts\/generate-npm-shrinkwrap\.mjs|extensions\/[^/]+\/(?:package\.json|npm-shrinkwrap\.json))$/u; +const PROMPT_SNAPSHOT_CHECK_PATH_RE = + /^(?:scripts\/(?:generate-prompt-snapshots\.ts|prompt-snapshot-files\.ts|sync-codex-model-prompt-fixture\.ts)|test\/helpers\/agents\/(?:happy-path-prompt-snapshots|prompt-snapshot-paths)\.ts|test\/fixtures\/agents\/prompt-snapshots\/.+)$/u; +const PROMPT_SNAPSHOT_OWNER_TEST_PATH_RE = + /^(?:scripts\/(?:generate-prompt-snapshots\.ts|prompt-snapshot-files\.ts|sync-codex-model-prompt-fixture\.ts)|test\/helpers\/agents\/(?:happy-path-prompt-snapshots|prompt-snapshot-paths)\.ts|test\/fixtures\/agents\/prompt-snapshots\/codex-model-catalog\/.+)$/u; const CORE_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.core.json"; const TARGETED_CORE_LINT_PATH_LIMIT = 8; const LINTABLE_CORE_PATH_RE = /^(?:src|ui|packages)\/.+\.[cm]?[jt]sx?$/u; @@ -173,6 +177,14 @@ export function shouldRunShrinkwrapGuard(paths) { return paths.some((changedPath) => SHRINKWRAP_POLICY_PATH_RE.test(changedPath)); } +export function shouldRunPromptSnapshotCheck(paths) { + return paths.some((changedPath) => PROMPT_SNAPSHOT_CHECK_PATH_RE.test(changedPath)); +} + +export function shouldRunPromptSnapshotOwnerTest(paths) { + return paths.some((changedPath) => PROMPT_SNAPSHOT_OWNER_TEST_PATH_RE.test(changedPath)); +} + export function shouldRunTestTempCreationReport(paths) { return paths.some((changedPath) => isChangedLaneTestPath(changedPath)); } @@ -259,6 +271,12 @@ export function createChangedCheckPlan(result, options = {}) { baseEnv, ); } + if (shouldRunPromptSnapshotCheck(result.paths)) { + add("prompt snapshot drift", ["prompt:snapshots:check"]); + } + if (shouldRunPromptSnapshotOwnerTest(result.paths)) { + add("prompt snapshot owner test", ["test:serial", "test/scripts/prompt-snapshots.test.ts"]); + } add("package patch guard", ["deps:patches:check"]); if (result.docsOnly) { diff --git a/scripts/sync-codex-model-prompt-fixture.ts b/scripts/sync-codex-model-prompt-fixture.ts index c55a61aef876..b0c36a395cb9 100644 --- a/scripts/sync-codex-model-prompt-fixture.ts +++ b/scripts/sync-codex-model-prompt-fixture.ts @@ -3,12 +3,12 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; +import { CODEX_MODEL_PROMPT_FIXTURE_DIR } from "../test/helpers/agents/prompt-snapshot-paths.js"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const PERSONALITY_PLACEHOLDER = "{{ personality }}"; -export const CODEX_MODEL_PROMPT_FIXTURE_DIR = - "test/fixtures/agents/prompt-snapshots/codex-model-catalog"; +export { CODEX_MODEL_PROMPT_FIXTURE_DIR }; type JsonObject = Record; type CodexPromptPersonality = "default" | "friendly" | "pragmatic"; diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index a58167892b41..b9b5e1be10c0 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -662,6 +662,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/check-dependency-pins.mjs", ["test/scripts/check-dependency-pins.test.ts"]], ["scripts/check-deadcode-unused-files.mjs", ["test/scripts/check-deadcode-unused-files.test.ts"]], ["scripts/check-dynamic-import-warts.mjs", ["test/scripts/check-dynamic-import-warts.test.ts"]], + ["scripts/generate-prompt-snapshots.ts", ["test/scripts/prompt-snapshots.test.ts"]], [ "scripts/lib/config-boundary-guard.mjs", [ @@ -1022,6 +1023,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/lib/official-external-plugin-catalog.json", OFFICIAL_EXTERNAL_CATALOG_TEST_TARGETS], ["scripts/lib/official-external-provider-catalog.json", OFFICIAL_EXTERNAL_CATALOG_TEST_TARGETS], ["scripts/lib/direct-run.mjs", ["test/scripts/changed-lanes.test.ts"]], + ["scripts/prompt-snapshot-files.ts", ["test/scripts/prompt-snapshots.test.ts"]], [ "scripts/docker/cleanup-smoke/Dockerfile", [...DOCKERFILE_CACHE_AND_DIGEST_TEST_TARGETS, "test/scripts/docker-build-helper.test.ts"], @@ -1154,6 +1156,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ "test/scripts/upgrade-survivor-config-recipe.test.ts", ], ], + ["scripts/sync-codex-model-prompt-fixture.ts", ["test/scripts/prompt-snapshots.test.ts"]], [ "scripts/lib/npm-pack-budget.mjs", ["test/release-check.test.ts", "test/scripts/test-install-sh-docker.test.ts"], @@ -1845,6 +1848,7 @@ const TEST_HELPER_NORMALIZE_TEXT_TARGETS = [ "src/auto-reply/status.test.ts", "src/tui/components/chat-log.test.ts", ]; +const HAPPY_PATH_PROMPT_SNAPSHOT_HELPER_TEST_TARGETS = ["test/scripts/prompt-snapshots.test.ts"]; const SOURCE_TEST_TARGETS = new Map([ ...PRECISE_SOURCE_TEST_TARGETS, ["src/test-utils/openclaw-test-state.ts", ["src/test-utils/openclaw-test-state.test.ts"]], @@ -1861,6 +1865,10 @@ const SOURCE_TEST_TARGETS = new Map([ CHANNEL_CONTRACT_REGISTRY_BACKED_TARGETS, ], ["test/helpers/normalize-text.ts", TEST_HELPER_NORMALIZE_TEXT_TARGETS], + [ + "test/helpers/agents/happy-path-prompt-snapshots.ts", + HAPPY_PATH_PROMPT_SNAPSHOT_HELPER_TEST_TARGETS, + ], ["ui/config/control-ui-chunking.ts", ["ui/src/ui/control-ui-chunking.test.ts"]], [ "src/plugin-sdk/test-helpers/directory-ids.ts", @@ -3086,10 +3094,19 @@ function shouldRouteChangedTargetWithoutImportGraph(changedPath) { ); } +function resolvePromptSnapshotFixtureTargets(changedPath) { + if (!/^test\/fixtures\/agents\/prompt-snapshots\/.+\.(?:json|md)$/u.test(changedPath)) { + return null; + } + return ["test/scripts/prompt-snapshots.test.ts"]; +} + function resolvePreciseChangedTestTargets(changedPath, options) { const cwd = options.cwd ?? process.cwd(); const mappedTargets = - resolveToolingTestTargets(changedPath) ?? SOURCE_TEST_TARGETS.get(changedPath); + resolveToolingTestTargets(changedPath) ?? + SOURCE_TEST_TARGETS.get(changedPath) ?? + resolvePromptSnapshotFixtureTargets(changedPath); if (mappedTargets) { return mappedTargets; } diff --git a/test/scripts/changed-lanes.test.ts b/test/scripts/changed-lanes.test.ts index 7e4e5323d9f8..9723b2e92385 100644 --- a/test/scripts/changed-lanes.test.ts +++ b/test/scripts/changed-lanes.test.ts @@ -19,6 +19,8 @@ import { createPnpmManagedCommand, createTargetedCoreLintCommand, shouldDelegateChangedCheckToCrabbox, + shouldRunPromptSnapshotCheck, + shouldRunPromptSnapshotOwnerTest, shouldRunShrinkwrapGuard, shouldRunTestTempCreationReport, createShrinkwrapGuardCommand, @@ -1295,6 +1297,45 @@ describe("scripts/changed-lanes", () => { expect(plan.commands.map((command) => command.args[0])).not.toContain("deps:shrinkwrap:check"); }); + it("runs prompt snapshot drift checks for prompt snapshot generator surfaces", () => { + expect( + shouldRunPromptSnapshotCheck([ + "scripts/generate-prompt-snapshots.ts", + "test/helpers/agents/happy-path-prompt-snapshots.ts", + "test/fixtures/agents/prompt-snapshots/runtime-happy-path/telegram-direct-codex-message-tool.md", + ]), + ).toBe(true); + + const result = detectChangedLanes(["test/helpers/agents/happy-path-prompt-snapshots.ts"]); + const plan = createChangedCheckPlan(result); + + expect(plan.commands).toContainEqual({ + name: "prompt snapshot drift", + args: ["prompt:snapshots:check"], + }); + expect(plan.commands).toContainEqual({ + name: "prompt snapshot owner test", + args: ["test:serial", "test/scripts/prompt-snapshots.test.ts"], + }); + }); + + it("runs the prompt snapshot owner test for model fixture generator surfaces", () => { + expect( + shouldRunPromptSnapshotOwnerTest([ + "scripts/sync-codex-model-prompt-fixture.ts", + "test/fixtures/agents/prompt-snapshots/codex-model-catalog/gpt-5.5.pragmatic.source.json", + ]), + ).toBe(true); + + const result = detectChangedLanes(["scripts/sync-codex-model-prompt-fixture.ts"]); + const plan = createChangedCheckPlan(result); + + expect(plan.commands).toContainEqual({ + name: "prompt snapshot owner test", + args: ["test:serial", "test/scripts/prompt-snapshots.test.ts"], + }); + }); + it("guards release metadata package changes to the top-level version field", () => { const dir = makeTempRepoRoot(tempDirs, "openclaw-release-metadata-"); git(dir, ["init", "-q", "--initial-branch=main"]); diff --git a/test/scripts/prompt-snapshots.test.ts b/test/scripts/prompt-snapshots.test.ts index 5cbff602c8bb..ab4ffa18260e 100644 --- a/test/scripts/prompt-snapshots.test.ts +++ b/test/scripts/prompt-snapshots.test.ts @@ -4,8 +4,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { createFormattedPromptSnapshotFiles } from "../../scripts/generate-prompt-snapshots.js"; import { deleteStalePromptSnapshotFiles } from "../../scripts/prompt-snapshot-files.js"; import { + CODEX_MODEL_PROMPT_FIXTURE_DIR as SYNC_CODEX_MODEL_PROMPT_FIXTURE_DIR, defaultCatalogPathCandidates, findDefaultCatalogPath, renderCodexModelInstructions, @@ -104,6 +106,10 @@ function listFindCommittedPromptSnapshotFiles(): string[] | null { } describe("happy path prompt snapshots", () => { + it("loads the generator entrypoint used by the prompt snapshot check", () => { + expect(createFormattedPromptSnapshotFiles).toEqual(expect.any(Function)); + }); + it("lists committed Codex prompt snapshot artifacts without scanning directories in-process", () => { expectNoReaddirSyncDuring(() => { const committed = listCommittedPromptSnapshotFiles(); @@ -213,6 +219,7 @@ describe("happy path prompt snapshots", () => { }); it("keeps the Codex model prompt fixture next to its source metadata", () => { + expect(SYNC_CODEX_MODEL_PROMPT_FIXTURE_DIR).toBe(CODEX_MODEL_PROMPT_FIXTURE_DIR); expect( fs.existsSync(path.join(CODEX_MODEL_PROMPT_FIXTURE_DIR, "gpt-5.5.pragmatic.instructions.md")), ).toBe(true); @@ -295,4 +302,59 @@ describe("happy path prompt snapshots", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + it("writes Codex model prompt fixtures from an explicit catalog", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-codex-catalog-write-")); + try { + const catalogPath = path.join(root, "models_cache.json"); + const outputDir = path.join(root, "out"); + fs.writeFileSync( + catalogPath, + JSON.stringify({ + models: [ + { + slug: "gpt-5.5", + model_messages: { + instructions_template: "System\n{{ personality }}\nEnd", + instructions_variables: { + personality_pragmatic: "Use terse engineering judgement.", + }, + }, + }, + ], + }), + ); + + const result = await runCodexModelPromptFixtureSync([ + "--catalog", + catalogPath, + "--source-label", + "", + "--catalog-git-head", + "abc123", + "--out-dir", + outputDir, + ]); + + expect(result.status).toBe("written"); + expect( + fs.readFileSync(path.join(outputDir, "gpt-5.5.pragmatic.instructions.md"), "utf8"), + ).toBe("System\nUse terse engineering judgement.\nEnd\n"); + expect( + JSON.parse(fs.readFileSync(path.join(outputDir, "gpt-5.5.pragmatic.source.json"), "utf8")), + ).toEqual({ + model: "gpt-5.5", + personality: "pragmatic", + source: { + catalogPath: "", + catalogKind: "models_cache", + catalogGitHead: "abc123", + field: + "model_messages.instructions_template + model_messages.instructions_variables.personality_pragmatic", + }, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index e1e507919369..94235942b010 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -3407,6 +3407,22 @@ describe("scripts/test-projects changed-target routing", () => { expect(repoSourceReads.length).toBeLessThan(100); }); + it("routes prompt snapshot generator helper edits to the owner test", () => { + for (const target of [ + "scripts/generate-prompt-snapshots.ts", + "scripts/prompt-snapshot-files.ts", + "scripts/sync-codex-model-prompt-fixture.ts", + "test/helpers/agents/happy-path-prompt-snapshots.ts", + "test/fixtures/agents/prompt-snapshots/codex-model-catalog/gpt-5.5.pragmatic.source.json", + "test/fixtures/agents/prompt-snapshots/codex-runtime-happy-path/telegram-direct-codex-message-tool.md", + ]) { + expect(resolveChangedTestTargetPlan([target])).toEqual({ + mode: "targets", + targets: ["test/scripts/prompt-snapshots.test.ts"], + }); + } + }); + it.each([ "test/vitest/vitest.agents-core.config.ts", "test/vitest/vitest.agents-embedded-agent.config.ts",