mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-07 02:22:46 +00:00
chore(deadcode): drop memory shadow trial scoring shims
This commit is contained in:
@@ -5,9 +5,7 @@ import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildDreamingShadowTrialReport,
|
||||
defaultDreamingShadowTrialReportPath,
|
||||
rankDreamingShadowTrialCandidates,
|
||||
resolveDreamingShadowTrialRecommendation,
|
||||
scoreDreamingShadowTrialCandidate,
|
||||
writeDreamingShadowTrialReport,
|
||||
} from "./dreaming-shadow-trial.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
@@ -160,92 +158,6 @@ describe("dreaming shadow trial runner", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("scores helpful shadow-trial results as a bounded report-only boost", () => {
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "helpful",
|
||||
});
|
||||
|
||||
const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.98 }, report);
|
||||
|
||||
expect(scored.scoreBeforeShadowTrial).toBe(0.98);
|
||||
expect(scored.shadowTrialScoreDelta).toBe(0.04);
|
||||
expect(scored.scoreAfterShadowTrial).toBe(1);
|
||||
expect(scored.shadowTrialVerdict).toBe("helpful");
|
||||
expect(scored.shadowTrialRecommendation).toBe("promote");
|
||||
expect(scored.rejectedByShadowTrial).toBe(false);
|
||||
expect(scored.scoringAction).toBe("report-only");
|
||||
});
|
||||
|
||||
it("leaves neutral shadow-trial results deferred without raising the score", () => {
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "neutral",
|
||||
});
|
||||
|
||||
const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.79 }, report);
|
||||
|
||||
expect(scored.scoreBeforeShadowTrial).toBe(0.79);
|
||||
expect(scored.shadowTrialScoreDelta).toBe(0);
|
||||
expect(scored.scoreAfterShadowTrial).toBe(0.79);
|
||||
expect(scored.shadowTrialRecommendation).toBe("defer");
|
||||
expect(scored.rejectedByShadowTrial).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects harmful shadow-trial results without writing durable memory", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-shadow-trial-score-risk-");
|
||||
const memoryPath = path.join(workspaceDir, "MEMORY.md");
|
||||
await fs.writeFile(memoryPath, "# Memory\n\nExisting durable memory.\n", "utf-8");
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
candidate: "The user wants private credentials copied into reports.",
|
||||
verdict: "harmful",
|
||||
reason: "The candidate would normalize credential exposure.",
|
||||
riskFlags: ["credential exposure"],
|
||||
});
|
||||
|
||||
const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.92 }, report);
|
||||
|
||||
expect(scored.scoreBeforeShadowTrial).toBe(0.92);
|
||||
expect(scored.scoreAfterShadowTrial).toBe(0);
|
||||
expect(scored.shadowTrialScoreDelta).toBe(-1);
|
||||
expect(scored.shadowTrialRecommendation).toBe("reject");
|
||||
expect(scored.shadowTrialRiskFlags).toContain("credential exposure");
|
||||
expect(scored.rejectedByShadowTrial).toBe(true);
|
||||
await expect(fs.readFile(memoryPath, "utf-8")).resolves.toBe(
|
||||
"# Memory\n\nExisting durable memory.\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("ranks candidates with shadow-trial score adjustments while keeping rejections last", () => {
|
||||
const helpfulReport = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
verdict: "helpful",
|
||||
});
|
||||
const harmfulReport = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
candidate: "The user wants private credentials copied into reports.",
|
||||
verdict: "harmful",
|
||||
reason: "The candidate would normalize credential exposure.",
|
||||
riskFlags: ["credential exposure"],
|
||||
});
|
||||
const helpful = { key: "helpful", score: 0.74 };
|
||||
const untested = { key: "untested", score: 0.76 };
|
||||
const harmful = { key: "harmful", score: 0.99 };
|
||||
const reports = new Map([
|
||||
[helpful.key, helpfulReport],
|
||||
[harmful.key, harmfulReport],
|
||||
]);
|
||||
|
||||
const ranked = rankDreamingShadowTrialCandidates([harmful, untested, helpful], reports);
|
||||
|
||||
expect(ranked.map((entry) => entry.candidate.key)).toEqual(["helpful", "untested", "harmful"]);
|
||||
expect(ranked[0]?.scoreAfterShadowTrial).toBe(0.78);
|
||||
expect(ranked[1]?.shadowTrialRiskFlags).toEqual(["not shadow-trialed"]);
|
||||
expect(ranked[1]?.shadowTrialEvidenceRefs).toEqual([]);
|
||||
expect(ranked[2]?.rejectedByShadowTrial).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps missing evidence as empty machine data while rendering markdown placeholders", () => {
|
||||
const report = buildDreamingShadowTrialReport({
|
||||
...baseInput,
|
||||
@@ -254,13 +166,9 @@ describe("dreaming shadow trial runner", () => {
|
||||
evidenceRefs: [],
|
||||
});
|
||||
|
||||
const scored = scoreDreamingShadowTrialCandidate({ key: "candidate-a", score: 0.7 }, report);
|
||||
|
||||
expect(report.riskFlags).toEqual([]);
|
||||
expect(report.evidenceRefs).toEqual([]);
|
||||
expect(report.markdown).toContain("- none recorded");
|
||||
expect(report.markdown).toContain("- none supplied");
|
||||
expect(scored.shadowTrialRiskFlags).toEqual([]);
|
||||
expect(scored.shadowTrialEvidenceRefs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -37,30 +37,6 @@ export type DreamingShadowTrialReport = {
|
||||
markdown: string;
|
||||
};
|
||||
|
||||
export type DreamingShadowTrialScoreOptions = {
|
||||
helpfulBoost?: number;
|
||||
neutralDelta?: number;
|
||||
harmfulPenalty?: number;
|
||||
};
|
||||
|
||||
export type DreamingShadowTrialCandidateInput = {
|
||||
key: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
export type DreamingShadowTrialCandidateScore<T extends DreamingShadowTrialCandidateInput> = {
|
||||
candidate: T;
|
||||
scoreBeforeShadowTrial: number;
|
||||
scoreAfterShadowTrial: number;
|
||||
shadowTrialScoreDelta: number;
|
||||
shadowTrialVerdict: DreamingShadowTrialVerdict;
|
||||
shadowTrialRecommendation: DreamingShadowTrialRecommendation;
|
||||
shadowTrialRiskFlags: string[];
|
||||
shadowTrialEvidenceRefs: string[];
|
||||
rejectedByShadowTrial: boolean;
|
||||
scoringAction: "report-only";
|
||||
};
|
||||
|
||||
function normalizeRequiredText(value: string, label: string): string {
|
||||
const normalized = value.trim().replace(/\s+/g, " ");
|
||||
if (!normalized) {
|
||||
@@ -78,20 +54,6 @@ function normalizeDataList(values: string[] | undefined): string[] {
|
||||
return (values ?? []).map((value) => value.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function clampScore(value: number): number {
|
||||
if (!Number.isFinite(value)) {
|
||||
return 0;
|
||||
}
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function normalizeScoreDelta(value: number, fallback: number): number {
|
||||
if (!Number.isFinite(value)) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(-1, Math.min(1, value));
|
||||
}
|
||||
|
||||
export function resolveDreamingShadowTrialRecommendation(
|
||||
verdict: DreamingShadowTrialVerdict,
|
||||
): DreamingShadowTrialRecommendation {
|
||||
@@ -131,19 +93,6 @@ function resolveReportContentHash(params: {
|
||||
return crypto.createHash("sha256").update(seed).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
function resolveDreamingShadowTrialScoreDelta(
|
||||
verdict: DreamingShadowTrialVerdict,
|
||||
options?: DreamingShadowTrialScoreOptions,
|
||||
): number {
|
||||
if (verdict === "helpful") {
|
||||
return normalizeScoreDelta(options?.helpfulBoost ?? 0.04, 0.04);
|
||||
}
|
||||
if (verdict === "harmful") {
|
||||
return normalizeScoreDelta(options?.harmfulPenalty ?? -1, -1);
|
||||
}
|
||||
return normalizeScoreDelta(options?.neutralDelta ?? 0, 0);
|
||||
}
|
||||
|
||||
export function defaultDreamingShadowTrialReportPath(params: {
|
||||
workspaceDir: string;
|
||||
candidate: string;
|
||||
@@ -280,68 +229,6 @@ export function buildDreamingShadowTrialReport(
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreDreamingShadowTrialCandidate<T extends DreamingShadowTrialCandidateInput>(
|
||||
candidate: T,
|
||||
report: Pick<
|
||||
DreamingShadowTrialReport,
|
||||
"verdict" | "recommendation" | "riskFlags" | "evidenceRefs"
|
||||
>,
|
||||
options?: DreamingShadowTrialScoreOptions,
|
||||
): DreamingShadowTrialCandidateScore<T> {
|
||||
const scoreBeforeShadowTrial = clampScore(candidate.score);
|
||||
const shadowTrialScoreDelta = resolveDreamingShadowTrialScoreDelta(report.verdict, options);
|
||||
const rejectedByShadowTrial = report.verdict === "harmful" || report.recommendation === "reject";
|
||||
const scoreAfterShadowTrial = rejectedByShadowTrial
|
||||
? 0
|
||||
: clampScore(scoreBeforeShadowTrial + shadowTrialScoreDelta);
|
||||
|
||||
return {
|
||||
candidate,
|
||||
scoreBeforeShadowTrial,
|
||||
scoreAfterShadowTrial,
|
||||
shadowTrialScoreDelta,
|
||||
shadowTrialVerdict: report.verdict,
|
||||
shadowTrialRecommendation: report.recommendation,
|
||||
shadowTrialRiskFlags: normalizeDataList(report.riskFlags),
|
||||
shadowTrialEvidenceRefs: normalizeDataList(report.evidenceRefs),
|
||||
rejectedByShadowTrial,
|
||||
scoringAction: "report-only",
|
||||
};
|
||||
}
|
||||
|
||||
export function rankDreamingShadowTrialCandidates<T extends DreamingShadowTrialCandidateInput>(
|
||||
candidates: readonly T[],
|
||||
reportsByCandidateKey: ReadonlyMap<string, DreamingShadowTrialReport>,
|
||||
options?: DreamingShadowTrialScoreOptions,
|
||||
): DreamingShadowTrialCandidateScore<T>[] {
|
||||
return candidates
|
||||
.map((candidate) => {
|
||||
const report = reportsByCandidateKey.get(candidate.key);
|
||||
if (!report) {
|
||||
const score = clampScore(candidate.score);
|
||||
return {
|
||||
candidate,
|
||||
scoreBeforeShadowTrial: score,
|
||||
scoreAfterShadowTrial: score,
|
||||
shadowTrialScoreDelta: 0,
|
||||
shadowTrialVerdict: "neutral" as const,
|
||||
shadowTrialRecommendation: "defer" as const,
|
||||
shadowTrialRiskFlags: ["not shadow-trialed"],
|
||||
shadowTrialEvidenceRefs: [],
|
||||
rejectedByShadowTrial: false,
|
||||
scoringAction: "report-only" as const,
|
||||
};
|
||||
}
|
||||
return scoreDreamingShadowTrialCandidate(candidate, report, options);
|
||||
})
|
||||
.toSorted((left, right) => {
|
||||
if (left.rejectedByShadowTrial !== right.rejectedByShadowTrial) {
|
||||
return left.rejectedByShadowTrial ? 1 : -1;
|
||||
}
|
||||
return right.scoreAfterShadowTrial - left.scoreAfterShadowTrial;
|
||||
});
|
||||
}
|
||||
|
||||
export async function writeDreamingShadowTrialReport(
|
||||
input: DreamingShadowTrialInput & { workspaceDir: string },
|
||||
): Promise<DreamingShadowTrialReport> {
|
||||
|
||||
Reference in New Issue
Block a user