diff --git a/extensions/memory-core/src/dreaming-shadow-trial.test.ts b/extensions/memory-core/src/dreaming-shadow-trial.test.ts index dc57d3441e2f..4f771655be10 100644 --- a/extensions/memory-core/src/dreaming-shadow-trial.test.ts +++ b/extensions/memory-core/src/dreaming-shadow-trial.test.ts @@ -5,9 +5,7 @@ import { describe, expect, it } from "vitest"; import { buildDreamingShadowTrialReport, defaultDreamingShadowTrialReportPath, - rankDreamingShadowTrialCandidates, resolveDreamingShadowTrialRecommendation, - scoreDreamingShadowTrialCandidate, writeDreamingShadowTrialReport, } from "./dreaming-shadow-trial.js"; import { createMemoryCoreTestHarness } from "./test-helpers.js"; @@ -160,92 +158,6 @@ describe("dreaming shadow trial runner", () => { }); }); - it("scores helpful shadow-trial results as a bounded report-only boost", () => { - const report = buildDreamingShadowTrialReport({ - ...baseInput, - verdict: "helpful", - }); - - const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.98 }, report); - - expect(scored.scoreBeforeShadowTrial).toBe(0.98); - expect(scored.shadowTrialScoreDelta).toBe(0.04); - expect(scored.scoreAfterShadowTrial).toBe(1); - expect(scored.shadowTrialVerdict).toBe("helpful"); - expect(scored.shadowTrialRecommendation).toBe("promote"); - expect(scored.rejectedByShadowTrial).toBe(false); - expect(scored.scoringAction).toBe("report-only"); - }); - - it("leaves neutral shadow-trial results deferred without raising the score", () => { - const report = buildDreamingShadowTrialReport({ - ...baseInput, - verdict: "neutral", - }); - - const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.79 }, report); - - expect(scored.scoreBeforeShadowTrial).toBe(0.79); - expect(scored.shadowTrialScoreDelta).toBe(0); - expect(scored.scoreAfterShadowTrial).toBe(0.79); - expect(scored.shadowTrialRecommendation).toBe("defer"); - expect(scored.rejectedByShadowTrial).toBe(false); - }); - - it("rejects harmful shadow-trial results without writing durable memory", async () => { - const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-score-risk-"); - const memoryPath = path.join(workspaceDir, "MEMORY.md"); - await fs.writeFile(memoryPath, "# Memory\n\nExisting durable memory.\n", "utf-8"); - const report = buildDreamingShadowTrialReport({ - ...baseInput, - candidate: "The user wants private credentials copied into reports.", - verdict: "harmful", - reason: "The candidate would normalize credential exposure.", - riskFlags: ["credential exposure"], - }); - - const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.92 }, report); - - expect(scored.scoreBeforeShadowTrial).toBe(0.92); - expect(scored.scoreAfterShadowTrial).toBe(0); - expect(scored.shadowTrialScoreDelta).toBe(-1); - expect(scored.shadowTrialRecommendation).toBe("reject"); - expect(scored.shadowTrialRiskFlags).toContain("credential exposure"); - expect(scored.rejectedByShadowTrial).toBe(true); - await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe( - "# Memory\n\nExisting durable memory.\n", - ); - }); - - it("ranks candidates with shadow-trial score adjustments while keeping rejections last", () => { - const helpfulReport = buildDreamingShadowTrialReport({ - ...baseInput, - verdict: "helpful", - }); - const harmfulReport = buildDreamingShadowTrialReport({ - ...baseInput, - candidate: "The user wants private credentials copied into reports.", - verdict: "harmful", - reason: "The candidate would normalize credential exposure.", - riskFlags: ["credential exposure"], - }); - const helpful = { key: "helpful", score: 0.74 }; - const untested = { key: "untested", score: 0.76 }; - const harmful = { key: "harmful", score: 0.99 }; - const reports = new Map([ - [helpful.key, helpfulReport], - [harmful.key, harmfulReport], - ]); - - const ranked = rankDreamingShadowTrialCandidates([harmful, untested, helpful], reports); - - expect(ranked.map((entry) => entry.candidate.key)).toEqual(["helpful", "untested", "harmful"]); - expect(ranked[0]?.scoreAfterShadowTrial).toBe(0.78); - expect(ranked[1]?.shadowTrialRiskFlags).toEqual(["not shadow-trialed"]); - expect(ranked[1]?.shadowTrialEvidenceRefs).toEqual([]); - expect(ranked[2]?.rejectedByShadowTrial).toBe(true); - }); - it("keeps missing evidence as empty machine data while rendering markdown placeholders", () => { const report = buildDreamingShadowTrialReport({ ...baseInput, @@ -254,13 +166,9 @@ describe("dreaming shadow trial runner", () => { evidenceRefs: [], }); - const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.7 }, report); - expect(report.riskFlags).toEqual([]); expect(report.evidenceRefs).toEqual([]); expect(report.markdown).toContain("- none recorded"); expect(report.markdown).toContain("- none supplied"); - expect(scored.shadowTrialRiskFlags).toEqual([]); - expect(scored.shadowTrialEvidenceRefs).toEqual([]); }); }); diff --git a/extensions/memory-core/src/dreaming-shadow-trial.ts b/extensions/memory-core/src/dreaming-shadow-trial.ts index a9f0127cc42c..799aa657c78b 100644 --- a/extensions/memory-core/src/dreaming-shadow-trial.ts +++ b/extensions/memory-core/src/dreaming-shadow-trial.ts @@ -37,30 +37,6 @@ export type DreamingShadowTrialReport = { markdown: string; }; -export type DreamingShadowTrialScoreOptions = { - helpfulBoost?: number; - neutralDelta?: number; - harmfulPenalty?: number; -}; - -export type DreamingShadowTrialCandidateInput = { - key: string; - score: number; -}; - -export type DreamingShadowTrialCandidateScore = { - candidate: T; - scoreBeforeShadowTrial: number; - scoreAfterShadowTrial: number; - shadowTrialScoreDelta: number; - shadowTrialVerdict: DreamingShadowTrialVerdict; - shadowTrialRecommendation: DreamingShadowTrialRecommendation; - shadowTrialRiskFlags: string[]; - shadowTrialEvidenceRefs: string[]; - rejectedByShadowTrial: boolean; - scoringAction: "report-only"; -}; - function normalizeRequiredText(value: string, label: string): string { const normalized = value.trim().replace(/\s+/g, " "); if (!normalized) { @@ -78,20 +54,6 @@ function normalizeDataList(values: string[] | undefined): string[] { return (values ?? []).map((value) => value.trim()).filter(Boolean); } -function clampScore(value: number): number { - if (!Number.isFinite(value)) { - return 0; - } - return Math.max(0, Math.min(1, value)); -} - -function normalizeScoreDelta(value: number, fallback: number): number { - if (!Number.isFinite(value)) { - return fallback; - } - return Math.max(-1, Math.min(1, value)); -} - export function resolveDreamingShadowTrialRecommendation( verdict: DreamingShadowTrialVerdict, ): DreamingShadowTrialRecommendation { @@ -131,19 +93,6 @@ function resolveReportContentHash(params: { return crypto.createHash("sha256").update(seed).digest("hex").slice(0, 12); } -function resolveDreamingShadowTrialScoreDelta( - verdict: DreamingShadowTrialVerdict, - options?: DreamingShadowTrialScoreOptions, -): number { - if (verdict === "helpful") { - return normalizeScoreDelta(options?.helpfulBoost ?? 0.04, 0.04); - } - if (verdict === "harmful") { - return normalizeScoreDelta(options?.harmfulPenalty ?? -1, -1); - } - return normalizeScoreDelta(options?.neutralDelta ?? 0, 0); -} - export function defaultDreamingShadowTrialReportPath(params: { workspaceDir: string; candidate: string; @@ -280,68 +229,6 @@ export function buildDreamingShadowTrialReport( }; } -export function scoreDreamingShadowTrialCandidate( - candidate: T, - report: Pick< - DreamingShadowTrialReport, - "verdict" | "recommendation" | "riskFlags" | "evidenceRefs" - >, - options?: DreamingShadowTrialScoreOptions, -): DreamingShadowTrialCandidateScore { - const scoreBeforeShadowTrial = clampScore(candidate.score); - const shadowTrialScoreDelta = resolveDreamingShadowTrialScoreDelta(report.verdict, options); - const rejectedByShadowTrial = report.verdict === "harmful" || report.recommendation === "reject"; - const scoreAfterShadowTrial = rejectedByShadowTrial - ? 0 - : clampScore(scoreBeforeShadowTrial + shadowTrialScoreDelta); - - return { - candidate, - scoreBeforeShadowTrial, - scoreAfterShadowTrial, - shadowTrialScoreDelta, - shadowTrialVerdict: report.verdict, - shadowTrialRecommendation: report.recommendation, - shadowTrialRiskFlags: normalizeDataList(report.riskFlags), - shadowTrialEvidenceRefs: normalizeDataList(report.evidenceRefs), - rejectedByShadowTrial, - scoringAction: "report-only", - }; -} - -export function rankDreamingShadowTrialCandidates( - candidates: readonly T[], - reportsByCandidateKey: ReadonlyMap, - options?: DreamingShadowTrialScoreOptions, -): DreamingShadowTrialCandidateScore[] { - return candidates - .map((candidate) => { - const report = reportsByCandidateKey.get(candidate.key); - if (!report) { - const score = clampScore(candidate.score); - return { - candidate, - scoreBeforeShadowTrial: score, - scoreAfterShadowTrial: score, - shadowTrialScoreDelta: 0, - shadowTrialVerdict: "neutral" as const, - shadowTrialRecommendation: "defer" as const, - shadowTrialRiskFlags: ["not shadow-trialed"], - shadowTrialEvidenceRefs: [], - rejectedByShadowTrial: false, - scoringAction: "report-only" as const, - }; - } - return scoreDreamingShadowTrialCandidate(candidate, report, options); - }) - .toSorted((left, right) => { - if (left.rejectedByShadowTrial !== right.rejectedByShadowTrial) { - return left.rejectedByShadowTrial ? 1 : -1; - } - return right.scoreAfterShadowTrial - left.scoreAfterShadowTrial; - }); -} - export async function writeDreamingShadowTrialReport( input: DreamingShadowTrialInput & { workspaceDir: string }, ): Promise {