From 2d25a09c0f50eddf74dc3051b399b188a9897b53 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:06:39 +0800 Subject: [PATCH] fix(native)!: recognize identical function evidence (#8442) * fix(native)!: recognize identical function evidence * test(native): verify same-evidence dimension citations * fix(compare): reject impossible same-evidence changes A comparison of one Evidence record cannot report a changed dimension, unequal subject hashes, or unequal digests on an unchanged dimension. * test(compare): admit the partial same-evidence fixture explicitly --------- Co-authored-by: Cursor Agent --- docs/guides/native/native-investigation.md | 5 + src/contracts/sessionToolContracts.ts | 2 +- src/domain/comparisonEvidenceResult.test.ts | 108 +++++++++ src/domain/comparisonEvidenceResult.ts | 34 +++ src/domain/functionComparison.ts | 3 +- src/domain/functionComparisonDimensions.ts | 4 +- src/domain/functionComparisonNormalization.ts | 10 +- src/domain/functionComparisonSchemas.ts | 6 + .../mcp/sameFunctionObservation.test.ts | 211 ++++++++++++++++++ 9 files changed, 378 insertions(+), 5 deletions(-) create mode 100644 src/domain/comparisonEvidenceResult.test.ts create mode 100644 tests/boundary/mcp/sameFunctionObservation.test.ts diff --git a/docs/guides/native/native-investigation.md b/docs/guides/native/native-investigation.md index 7cac7044f5..717714e319 100644 --- a/docs/guides/native/native-investigation.md +++ b/docs/guides/native/native-investigation.md @@ -1279,6 +1279,11 @@ and slice commitments establish the same canonical entry. The result reports remain separate dimensions. Equal numeric addresses without applicable coordinate commitments do not establish same-entry correspondence. Inspect the retained source Evidence and coverage when correspondence remains ambiguous. +When both inputs authenticate to the same Evidence ID and an applicable entry +comparison is unavailable, the match reports `method: "same-evidence"`. +This identifies one selected observation, not a correspondence between distinct +records or proof of complete analysis. Unavailable facets and unestablished +call coverage remain unknown. To aggregate already-retained comparison results, pass their exact IDs to `find_changed_behavior`, for example: diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 67a97ddecc..0230c7cb17 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -123,7 +123,7 @@ export const SESSION_TOOL_CONTRACTS = [ ), session( "compare_functions", - "Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Authenticated same-artifact entry coordinates establish correspondence independently of renamed labels; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", + "Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. The same authenticated Evidence record or compatible same-artifact entry coordinates establish correspondence independently of symbol origin; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.", functionComparisonInputSchema, ), session( diff --git a/src/domain/comparisonEvidenceResult.test.ts b/src/domain/comparisonEvidenceResult.test.ts new file mode 100644 index 0000000000..7976cb7cb1 --- /dev/null +++ b/src/domain/comparisonEvidenceResult.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; + +import { readComparisonResult } from "./comparisonEvidenceResult.js"; +import type { Evidence } from "./evidence.js"; +import { + functionComparisonDimensionNameSchema, + functionComparisonResultSchema, +} from "./functionComparisonSchemas.js"; + +const source = `ev_${"a".repeat(64)}`; +const hash = "b".repeat(64); +const other = "c".repeat(64); + +const evidence = (result: unknown): Evidence => + ({ + evidence_id: `ev_${"d".repeat(64)}`, + evidence_links: [source], + parameters: { left_evidence_id: source, right_evidence_id: source }, + normalized_result: result, + }) as unknown as Evidence; + +const baseline = () => ({ + status: "unchanged" as const, + function_match: { + status: "matched" as const, + method: "same-evidence" as const, + left_name: "main", + right_name: "main", + }, + left_subject_sha256: hash, + right_subject_sha256: hash, + summary: { unchanged: 8, changed: 0, unknown: 0 }, + dimensions: functionComparisonDimensionNameSchema.options.map( + (dimension) => ({ + dimension, + left_count: 1, + right_count: 1, + evidence_links: [source, source], + limitations: [], + status: "unchanged" as const, + left_digest: hash, + right_digest: hash, + text_delta: null, + conclusion_kind: "derived_relationship" as const, + }), + ), + changes: [], + limitations: [], +}); + +describe("same-evidence comparison admission", () => { + it("accepts a consistent complete comparison", () => { + const result = baseline(); + expect(functionComparisonResultSchema.safeParse(result).success).toBe(true); + expect(readComparisonResult(evidence(result), "compare_functions").ok).toBe( + true, + ); + }); + + it("rejects a changed dimension for one source", () => { + const result = baseline(); + const changed = { + ...result.dimensions[0]!, + status: "changed" as const, + right_digest: other, + }; + const candidate = { + ...result, + status: "changed" as const, + summary: { unchanged: 7, changed: 1, unknown: 0 }, + dimensions: [changed, ...result.dimensions.slice(1)], + changes: [changed], + }; + expect(functionComparisonResultSchema.safeParse(candidate).success).toBe( + true, + ); + expect( + readComparisonResult(evidence(candidate), "compare_functions").ok, + ).toBe(false); + }); + + it("rejects unequal subject hashes for one source", () => { + const candidate = { ...baseline(), right_subject_sha256: other }; + expect(functionComparisonResultSchema.safeParse(candidate).success).toBe( + true, + ); + expect( + readComparisonResult(evidence(candidate), "compare_functions").ok, + ).toBe(false); + }); + + it("rejects unequal unchanged digests for one source", () => { + const result = baseline(); + const candidate = { + ...result, + dimensions: [ + { ...result.dimensions[0]!, right_digest: other }, + ...result.dimensions.slice(1), + ], + }; + expect(functionComparisonResultSchema.safeParse(candidate).success).toBe( + true, + ); + expect( + readComparisonResult(evidence(candidate), "compare_functions").ok, + ).toBe(false); + }); +}); diff --git a/src/domain/comparisonEvidenceResult.ts b/src/domain/comparisonEvidenceResult.ts index 58f6f5e619..0027acf47c 100644 --- a/src/domain/comparisonEvidenceResult.ts +++ b/src/domain/comparisonEvidenceResult.ts @@ -15,6 +15,7 @@ import { type ProcessCaptureComparison, } from "./process/processComparison.js"; import { err, ok, type Result } from "./result.js"; +import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js"; type ComparisonResults = { readonly compare_artifacts: ArtifactComparisonResult; @@ -58,6 +59,39 @@ export function readComparisonResult( evidence.normalized_result, ); if (!parsed.ok) return parsed; + if (parsed.value.function_match.method === "same-evidence") { + try { + const { left, right } = comparisonSourceEvidenceSides(evidence); + const sourceId = left[0]; + const sameSource = + left.length === 1 && + right.length === 1 && + sourceId === right[0] && + sourceId !== undefined && + evidence.evidence_links.includes(sourceId) && + parsed.value.dimensions.every(({ evidence_links }) => + evidence_links.every((id) => id === sourceId), + ); + const impossibleChange = + parsed.value.left_subject_sha256 !== + parsed.value.right_subject_sha256 || + parsed.value.dimensions.some( + (dimension) => + dimension.status === "changed" || + (dimension.status === "unchanged" && + dimension.left_digest !== dimension.right_digest), + ); + if (!sameSource || impossibleChange) + return err( + new EvidenceIntegrityError( + "Same-Evidence function comparison must cite the same source on both sides", + ), + ); + } catch (cause) { + if (cause instanceof EvidenceIntegrityError) return err(cause); + throw cause; + } + } const links = validateFindingLinks( evidence, parsed.value.dimensions.flatMap(({ evidence_links }) => evidence_links), diff --git a/src/domain/functionComparison.ts b/src/domain/functionComparison.ts index 3cdaf9c299..97a02858c2 100644 --- a/src/domain/functionComparison.ts +++ b/src/domain/functionComparison.ts @@ -26,7 +26,8 @@ export const compareFunctions = ( const dimensions = compareDimensions(left, right, { links, providersDiffer, - sameEntry: match.method === "same-entry", + sameIdentity: + match.method === "same-entry" || match.method === "same-evidence", coordinateContext, }); const changes = dimensions.filter(({ status }) => status !== "unchanged"); diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 230f2eae43..6c2f1d2b62 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -34,7 +34,7 @@ type ComparisonContext = Pick< DimensionResultInput, "links" | "providersDiffer" > & { - readonly sameEntry: boolean; + readonly sameIdentity: boolean; readonly coordinateContext: FunctionSnapshotCoordinateContext; }; @@ -559,7 +559,7 @@ const compareIdentity = ( context: ComparisonContext, ): FunctionDimension => { if ( - !context.sameEntry && + !context.sameIdentity && (!hasStableNativeSymbol( left.dossier.procedure.name, left.dossier.procedure.symbol, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 177be5d0bd..9a0d10ad93 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -49,13 +49,21 @@ export const functionMatch = ( left_name: leftName, right_name: rightName, }; + } + if (left.evidence.evidence_id === right.evidence.evidence_id) + return { + status: "matched", + method: "same-evidence", + left_name: leftName, + right_name: rightName, + }; + if (coordinateContext === "common") return { status: "ambiguous", method: "explicit", left_name: leftName, right_name: rightName, }; - } const stableNames = hasStableNativeSymbol(leftName, left.dossier.procedure.symbol) && hasStableNativeSymbol(rightName, right.dossier.procedure.symbol); diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 3670801901..4a38ac365e 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -85,6 +85,12 @@ const functionMatchSchema = z.union([ left_name: z.string(), right_name: z.string(), }), + z.object({ + status: z.literal("matched"), + method: z.literal("same-evidence"), + left_name: z.string(), + right_name: z.string(), + }), z.object({ status: z.enum(["mismatched", "ambiguous"]), method: z.literal("explicit"), diff --git a/tests/boundary/mcp/sameFunctionObservation.test.ts b/tests/boundary/mcp/sameFunctionObservation.test.ts new file mode 100644 index 0000000000..915c4e5d1a --- /dev/null +++ b/tests/boundary/mcp/sameFunctionObservation.test.ts @@ -0,0 +1,211 @@ +import { afterAll, expect, it } from "vitest"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchema } from "ajv"; +import type { Client } from "@modelcontextprotocol/client"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { functionDossierSchema } from "../../../src/domain/native/functionDossier.js"; +import { functionComparisonResultSchema } from "../../../src/domain/functionComparisonSchemas.js"; + +const example = FUNCTION_COMPARISON_EXAMPLE.left; +const dossier = functionDossierSchema.parse(example.normalized_result); +const { symbol: _symbol, ...procedure } = dossier.procedure; +const observation = (request: string) => + createEvidence( + { path: "/tmp/fixture", sha256: "0".repeat(64), format: "mach-o" }, + example.provider, + { + operation: "analyze_function", + parameters: { procedure: request }, + result: { + ...dossier, + procedure, + comments: [], + facet_availability: { + ...dossier.facet_availability, + comments: { available: false, reason: "Fixture has no comments." }, + }, + }, + confidence: "derived", + authority: "shipped-artifact", + relationshipCoverage: { status: "complete" }, + }, + ); +const main = observation("main"); +const another = observation("another-selector"); +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); +const ref = (evidence_id: string) => ({ + kind: "retained-evidence", + evidence_id, +}); + +const expectRejectedDimensionCitation = async ( + client: Client, + comparison: Evidence, +) => { + const original = functionComparisonResultSchema.parse( + comparison.normalized_result, + ); + const altered = (dimension: (typeof original.dimensions)[number]) => + dimension.dimension === "comments" + ? { + ...dimension, + evidence_links: [main.evidence_id, another.evidence_id], + } + : dimension; + const result = functionComparisonResultSchema.parse({ + ...original, + dimensions: original.dimensions.map(altered), + changes: original.changes.map(altered), + }); + const invalid = createEvidence(undefined, comparison.provider, { + predicateType: comparison.predicate_type, + operation: comparison.operation, + parameters: comparison.parameters, + result, + confidence: comparison.confidence, + authority: comparison.authority, + limitations: comparison.limitations, + evidenceLinks: [main.evidence_id, another.evidence_id], + }); + const response = await client.callTool({ + name: "find_changed_behavior", + arguments: { comparisons: [invalid] }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error.code).toBe( + "evidence_integrity_mismatch", + ); +}; + +it.each(["full", "compact"] as const)( + "compares the same authenticated observation while preserving incomplete facets (%s)", + async (profile) => { + const session = createTestBinarySession(() => { + throw new Error("offline comparison must not launch a provider"); + }); + expect(session.recordEvidence(main).ok).toBe(true); + expect(session.recordEvidence(another).ok).toBe(true); + const { client } = await createApplicationMcpHarness({ + session, + schemaValidator: schemaValidation, + serverOptions: { + environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: profile }, + }, + }); + const schema = (await client.listTools()).tools.find( + ({ name }) => name === "compare_functions", + )?.outputSchema; + if (schema === undefined) throw new Error("Missing output schema"); + const reply = await client.callTool({ + name: "compare_functions", + arguments: { left: ref(main.evidence_id), right: ref(main.evidence_id) }, + }); + expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true); + expect( + new Ajv2020({ strict: false }).validate( + schema as AnySchema, + reply.structuredContent, + ), + ).toBe(true); + const result = parseEvidence(reply.structuredContent).normalized_result; + expect(result).toMatchObject({ + status: "unknown", + function_match: { status: "matched", method: "same-evidence" }, + dimensions: expect.arrayContaining([ + expect.objectContaining({ dimension: "identity", status: "unchanged" }), + expect.objectContaining({ dimension: "comments", status: "unknown" }), + ]), + }); + const comparison = parseEvidence(reply.structuredContent); + const behavior = await client.callTool({ + name: "find_changed_behavior", + arguments: { comparisons: [ref(comparison.evidence_id)] }, + }); + expect(behavior.isError, JSON.stringify(behavior.content)).not.toBe(true); + const behaviorResult = parseEvidence( + behavior.structuredContent, + ).normalized_result; + expect(behaviorResult).toMatchObject({ + behavior_status: "unknown", + findings: { + items: expect.arrayContaining([ + expect.objectContaining({ dimension: "comments", status: "unknown" }), + ]), + }, + }); + expect(behaviorResult).toMatchObject({ + findings: { + items: expect.not.arrayContaining([ + expect.objectContaining({ dimension: "identity" }), + ]), + }, + }); + const contradictory = createEvidence(undefined, comparison.provider, { + predicateType: comparison.predicate_type, + operation: comparison.operation, + parameters: { + left_evidence_id: main.evidence_id, + right_evidence_id: another.evidence_id, + }, + result: comparison.normalized_result, + confidence: comparison.confidence, + authority: comparison.authority, + limitations: comparison.limitations, + evidenceLinks: [main.evidence_id, another.evidence_id], + }); + const contradictoryBehavior = await client.callTool({ + name: "find_changed_behavior", + arguments: { comparisons: [contradictory] }, + }); + expect(contradictoryBehavior.isError).toBe(true); + expect(parseMcpToolError(contradictoryBehavior).error.code).toBe( + "evidence_integrity_mismatch", + ); + await expectRejectedDimensionCitation(client, comparison); + const distinct = await client.callTool({ + name: "compare_functions", + arguments: { + left: ref(main.evidence_id), + right: ref(another.evidence_id), + }, + }); + expect( + parseEvidence(distinct.structuredContent).normalized_result, + ).toMatchObject({ + function_match: { status: "ambiguous" }, + }); + const rebound = await client.callTool({ + name: "compare_functions", + arguments: { + left: main, + right: { ...main, caller_input_path: "/tmp/alternate-spelling" }, + }, + }); + expect( + parseEvidence(rebound.structuredContent).normalized_result, + ).toMatchObject({ + function_match: { status: "matched", method: "same-evidence" }, + }); + const tampered = await client.callTool({ + name: "compare_functions", + arguments: { + left: ref(main.evidence_id), + right: { ...main, normalized_result: { ...dossier, procedure } }, + }, + }); + expect(tampered.isError).toBe(true); + expect(parseMcpToolError(tampered).error.code).toBe( + "evidence_integrity_mismatch", + ); + }, +);