fix(native)!: recognize identical function evidence
This commit is contained in:
1 parent
6e9ee97a69
commit
12d5954653
8 files changed
+215
-5
No files matched your search
@@ -1152,6 +1152,11 @@ and slice commitments establish the same canonical entry. The result reports
|
||||
remain separate dimensions. Equal numeric addresses without applicable coordinate
|
||||
commitments do not establish same-entry correspondence. Inspect the retained
|
||||
source Evidence and coverage when correspondence remains ambiguous.
|
||||
When both inputs authenticate to the same Evidence ID and an applicable entry
|
||||
comparison is unavailable, the match reports `method: "same-evidence"`.
|
||||
This identifies one selected observation, not a correspondence between distinct
|
||||
records or proof of complete analysis. Unavailable facets and unestablished
|
||||
call coverage remain unknown.
|
||||
To aggregate already-retained comparison results, pass their exact IDs to
|
||||
`find_changed_behavior`, for example:
|
||||
|
||||
|
||||
@@ -123,7 +123,7 @@ export const SESSION_TOOL_CONTRACTS = [
|
||||
),
|
||||
session(
|
||||
"compare_functions",
|
||||
"Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Authenticated same-artifact entry coordinates establish correspondence independently of renamed labels; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.",
|
||||
"Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. The same authenticated Evidence record or compatible same-artifact entry coordinates establish correspondence independently of symbol origin; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.",
|
||||
functionComparisonInputSchema,
|
||||
),
|
||||
session(
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
type ProcessCaptureComparison,
|
||||
} from "./process/processComparison.js";
|
||||
import { err, ok, type Result } from "./result.js";
|
||||
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
|
||||
|
||||
type ComparisonResults = {
|
||||
readonly compare_artifacts: ArtifactComparisonResult;
|
||||
@@ -58,6 +59,30 @@ export function readComparisonResult(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
if (!parsed.ok) return parsed;
|
||||
if (parsed.value.function_match.method === "same-evidence") {
|
||||
try {
|
||||
const { left, right } = comparisonSourceEvidenceSides(evidence);
|
||||
const sourceId = left[0];
|
||||
if (
|
||||
left.length !== 1 ||
|
||||
right.length !== 1 ||
|
||||
sourceId !== right[0] ||
|
||||
sourceId === undefined ||
|
||||
!evidence.evidence_links.includes(sourceId) ||
|
||||
parsed.value.dimensions.some(({ evidence_links }) =>
|
||||
evidence_links.some((id) => id !== sourceId),
|
||||
)
|
||||
)
|
||||
return err(
|
||||
new EvidenceIntegrityError(
|
||||
"Same-Evidence function comparison must cite the same source on both sides",
|
||||
),
|
||||
);
|
||||
} catch (cause) {
|
||||
if (cause instanceof EvidenceIntegrityError) return err(cause);
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
const links = validateFindingLinks(
|
||||
evidence,
|
||||
parsed.value.dimensions.flatMap(({ evidence_links }) => evidence_links),
|
||||
|
||||
@@ -26,7 +26,8 @@ export const compareFunctions = (
|
||||
const dimensions = compareDimensions(left, right, {
|
||||
links,
|
||||
providersDiffer,
|
||||
sameEntry: match.method === "same-entry",
|
||||
sameIdentity:
|
||||
match.method === "same-entry" || match.method === "same-evidence",
|
||||
coordinateContext,
|
||||
});
|
||||
const changes = dimensions.filter(({ status }) => status !== "unchanged");
|
||||
|
||||
@@ -34,7 +34,7 @@ type ComparisonContext = Pick<
|
||||
DimensionResultInput,
|
||||
"links" | "providersDiffer"
|
||||
> & {
|
||||
readonly sameEntry: boolean;
|
||||
readonly sameIdentity: boolean;
|
||||
readonly coordinateContext: FunctionSnapshotCoordinateContext;
|
||||
};
|
||||
|
||||
@@ -554,7 +554,7 @@ const compareIdentity = (
|
||||
context: ComparisonContext,
|
||||
): FunctionDimension => {
|
||||
if (
|
||||
!context.sameEntry &&
|
||||
!context.sameIdentity &&
|
||||
(!hasStableNativeSymbol(
|
||||
left.dossier.procedure.name,
|
||||
left.dossier.procedure.symbol,
|
||||
|
||||
@@ -49,13 +49,21 @@ export const functionMatch = (
|
||||
left_name: leftName,
|
||||
right_name: rightName,
|
||||
};
|
||||
}
|
||||
if (left.evidence.evidence_id === right.evidence.evidence_id)
|
||||
return {
|
||||
status: "matched",
|
||||
method: "same-evidence",
|
||||
left_name: leftName,
|
||||
right_name: rightName,
|
||||
};
|
||||
if (coordinateContext === "common")
|
||||
return {
|
||||
status: "ambiguous",
|
||||
method: "explicit",
|
||||
left_name: leftName,
|
||||
right_name: rightName,
|
||||
};
|
||||
}
|
||||
const stableNames =
|
||||
hasStableNativeSymbol(leftName, left.dossier.procedure.symbol) &&
|
||||
hasStableNativeSymbol(rightName, right.dossier.procedure.symbol);
|
||||
|
||||
@@ -79,6 +79,12 @@ const functionMatchSchema = z.union([
|
||||
left_name: z.string(),
|
||||
right_name: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
status: z.literal("matched"),
|
||||
method: z.literal("same-evidence"),
|
||||
left_name: z.string(),
|
||||
right_name: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
status: z.enum(["mismatched", "ambiguous"]),
|
||||
method: z.literal("explicit"),
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { afterAll, expect, it } from "vitest";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import type { AnySchema } from "ajv";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestBinarySession } from "../../fixtures/binarySession.js";
|
||||
import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { functionDossierSchema } from "../../../src/domain/native/functionDossier.js";
|
||||
|
||||
const example = FUNCTION_COMPARISON_EXAMPLE.left;
|
||||
const dossier = functionDossierSchema.parse(example.normalized_result);
|
||||
const { symbol: _symbol, ...procedure } = dossier.procedure;
|
||||
const observation = (request: string) =>
|
||||
createEvidence(
|
||||
{ path: "/tmp/fixture", sha256: "0".repeat(64), format: "mach-o" },
|
||||
example.provider,
|
||||
{
|
||||
operation: "analyze_function",
|
||||
parameters: { procedure: request },
|
||||
result: {
|
||||
...dossier,
|
||||
procedure,
|
||||
comments: [],
|
||||
facet_availability: {
|
||||
...dossier.facet_availability,
|
||||
comments: { available: false, reason: "Fixture has no comments." },
|
||||
},
|
||||
},
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
relationshipCoverage: { status: "complete" },
|
||||
},
|
||||
);
|
||||
const main = observation("main");
|
||||
const another = observation("another-selector");
|
||||
const schemaValidation = createMcpSchemaValidation();
|
||||
afterAll(() => schemaValidation.clear());
|
||||
const ref = (evidence_id: string) => ({
|
||||
kind: "retained-evidence",
|
||||
evidence_id,
|
||||
});
|
||||
|
||||
it.each(["full", "compact"] as const)(
|
||||
"compares the same authenticated observation while preserving incomplete facets (%s)",
|
||||
async (profile) => {
|
||||
const session = createTestBinarySession(() => {
|
||||
throw new Error("offline comparison must not launch a provider");
|
||||
});
|
||||
expect(session.recordEvidence(main).ok).toBe(true);
|
||||
expect(session.recordEvidence(another).ok).toBe(true);
|
||||
const { client } = await createApplicationMcpHarness({
|
||||
session,
|
||||
schemaValidator: schemaValidation,
|
||||
serverOptions: {
|
||||
environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: profile },
|
||||
},
|
||||
});
|
||||
const schema = (await client.listTools()).tools.find(
|
||||
({ name }) => name === "compare_functions",
|
||||
)?.outputSchema;
|
||||
if (schema === undefined) throw new Error("Missing output schema");
|
||||
const reply = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: { left: ref(main.evidence_id), right: ref(main.evidence_id) },
|
||||
});
|
||||
expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true);
|
||||
expect(
|
||||
new Ajv2020({ strict: false }).validate(
|
||||
schema as AnySchema,
|
||||
reply.structuredContent,
|
||||
),
|
||||
).toBe(true);
|
||||
const result = parseEvidence(reply.structuredContent).normalized_result;
|
||||
expect(result).toMatchObject({
|
||||
status: "unknown",
|
||||
function_match: { status: "matched", method: "same-evidence" },
|
||||
dimensions: expect.arrayContaining([
|
||||
expect.objectContaining({ dimension: "identity", status: "unchanged" }),
|
||||
expect.objectContaining({ dimension: "comments", status: "unknown" }),
|
||||
]),
|
||||
});
|
||||
const comparison = parseEvidence(reply.structuredContent);
|
||||
const behavior = await client.callTool({
|
||||
name: "find_changed_behavior",
|
||||
arguments: { comparisons: [ref(comparison.evidence_id)] },
|
||||
});
|
||||
expect(behavior.isError, JSON.stringify(behavior.content)).not.toBe(true);
|
||||
const behaviorResult = parseEvidence(
|
||||
behavior.structuredContent,
|
||||
).normalized_result;
|
||||
expect(behaviorResult).toMatchObject({
|
||||
behavior_status: "unknown",
|
||||
findings: {
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({ dimension: "comments", status: "unknown" }),
|
||||
]),
|
||||
},
|
||||
});
|
||||
expect(behaviorResult).toMatchObject({
|
||||
findings: {
|
||||
items: expect.not.arrayContaining([
|
||||
expect.objectContaining({ dimension: "identity" }),
|
||||
]),
|
||||
},
|
||||
});
|
||||
const contradictory = createEvidence(undefined, comparison.provider, {
|
||||
predicateType: comparison.predicate_type,
|
||||
operation: comparison.operation,
|
||||
parameters: {
|
||||
left_evidence_id: main.evidence_id,
|
||||
right_evidence_id: another.evidence_id,
|
||||
},
|
||||
result: comparison.normalized_result,
|
||||
confidence: comparison.confidence,
|
||||
authority: comparison.authority,
|
||||
limitations: comparison.limitations,
|
||||
evidenceLinks: [main.evidence_id, another.evidence_id],
|
||||
});
|
||||
const contradictoryBehavior = await client.callTool({
|
||||
name: "find_changed_behavior",
|
||||
arguments: { comparisons: [contradictory] },
|
||||
});
|
||||
expect(contradictoryBehavior.isError).toBe(true);
|
||||
expect(parseMcpToolError(contradictoryBehavior).error.code).toBe(
|
||||
"evidence_integrity_mismatch",
|
||||
);
|
||||
const distinct = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: {
|
||||
left: ref(main.evidence_id),
|
||||
right: ref(another.evidence_id),
|
||||
},
|
||||
});
|
||||
expect(
|
||||
parseEvidence(distinct.structuredContent).normalized_result,
|
||||
).toMatchObject({
|
||||
function_match: { status: "ambiguous" },
|
||||
});
|
||||
const rebound = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: {
|
||||
left: main,
|
||||
right: { ...main, caller_input_path: "/tmp/alternate-spelling" },
|
||||
},
|
||||
});
|
||||
expect(
|
||||
parseEvidence(rebound.structuredContent).normalized_result,
|
||||
).toMatchObject({
|
||||
function_match: { status: "matched", method: "same-evidence" },
|
||||
});
|
||||
const tampered = await client.callTool({
|
||||
name: "compare_functions",
|
||||
arguments: {
|
||||
left: ref(main.evidence_id),
|
||||
right: { ...main, normalized_result: { ...dossier, procedure } },
|
||||
},
|
||||
});
|
||||
expect(tampered.isError).toBe(true);
|
||||
expect(parseMcpToolError(tampered).error.code).toBe(
|
||||
"evidence_integrity_mismatch",
|
||||
);
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user