From 5f4e39fea263abca55fd57b9bb19b1e8dc54e0b2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:53:46 +0800 Subject: [PATCH] fix(comparison): validate function result projections Validate canonical dimension ownership, summary counts, and exact changed or unknown projections before investigation consumers publish derived Evidence. Reuse the shared dimension contract in selectors and keep historical source fixtures internally consistent. Verify authentic malformed comparisons across all three MCP consumers while preserving valid reordered and unknown results and retained source Evidence. --- src/domain/functionComparisonSchemas.ts | 54 +++- ...uctionVerification.functionSources.test.ts | 18 +- .../reconstructionVerificationSchemas.ts | 10 +- src/domain/staticRuntimeCorrelation.ts | 31 +-- .../functionComparisonIntegrityMcp.test.ts | 262 ++++++++++++++++++ 5 files changed, 338 insertions(+), 37 deletions(-) create mode 100644 tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 75957a7b70..7c4e1dfbb5 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -1,5 +1,6 @@ import { z } from "zod"; -import { overallStatus } from "./functionComparisonResults.js"; +import { overallStatus, summarize } from "./functionComparisonResults.js"; +import { canonicalJson } from "./comparisonSemantics.js"; import { evidenceInputSchema } from "./evidenceReference.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -91,11 +92,60 @@ export const functionComparisonResultSchema = z changed: z.int().min(0), unknown: z.int().min(0), }), - dimensions: z.array(functionDimensionSchema).length(8), + dimensions: z + .array(functionDimensionSchema) + .length(functionComparisonDimensionNameSchema.options.length), changes: z.array(functionDimensionSchema), limitations: z.array(z.string()), }) .superRefine((result, context) => { + const dimensions = new Map(); + for (const [index, dimension] of result.dimensions.entries()) { + if (dimensions.has(dimension.dimension)) + context.addIssue({ + code: "custom", + path: ["dimensions", index, "dimension"], + message: "Function comparison dimensions must occur exactly once.", + }); + dimensions.set(dimension.dimension, dimension); + } + const expectedSummary = summarize(result.dimensions); + for (const status of comparisonStatusSchema.options) + if (result.summary[status] !== expectedSummary[status]) + context.addIssue({ + code: "custom", + path: ["summary", status], + message: "Function comparison summary must count its dimensions.", + }); + const changes = new Set(); + for (const [index, change] of result.changes.entries()) { + const dimension = dimensions.get(change.dimension); + if ( + changes.has(change.dimension) || + dimension === undefined || + change.status === "unchanged" || + canonicalJson(change) !== canonicalJson(dimension) + ) + context.addIssue({ + code: "custom", + path: ["changes", index], + message: + "Function comparison changes must cite each changed or unknown dimension exactly as recorded.", + }); + changes.add(change.dimension); + } + if ( + result.dimensions.some( + ({ dimension, status }) => + status !== "unchanged" && !changes.has(dimension), + ) + ) + context.addIssue({ + code: "custom", + path: ["changes"], + message: + "Function comparison changes must include every changed or unknown dimension.", + }); if ( result.status !== overallStatus(result.dimensions, result.function_match.status) diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts index eadb957c40..c053730812 100644 --- a/src/domain/reconstructionVerification.functionSources.test.ts +++ b/src/domain/reconstructionVerification.functionSources.test.ts @@ -120,23 +120,27 @@ const comparison = ( target?: EvidenceSubjectTarget; } = {}, resultInput = baselineResult, -): Evidence => - createEvidence(identity.target, identity.provider ?? provider, { +): Evidence => { + const links = sources.map(({ evidence_id: id }) => id); + const dimensions = resultInput.dimensions.map((dimension) => ({ + ...dimension, + evidence_links: links, + })); + return createEvidence(identity.target, identity.provider ?? provider, { predicateType: "rea.function-comparison", operation: "compare_functions", parameters, result: jsonValueSchema.parse({ ...resultInput, - dimensions: resultInput.dimensions.map((dimension) => ({ - ...dimension, - evidence_links: sources.map(({ evidence_id: id }) => id), - })), + dimensions, + changes: dimensions.filter(({ status }) => status !== "unchanged"), }), confidence: "derived", authority: "analyst-inference", - evidenceLinks: sources.map(({ evidence_id: id }) => id), + evidenceLinks: links, ...observation, }); +}; const specification = (evidence: Evidence, dimension = "pseudocode") => ({ name: "Function reconstruction", diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.ts index 1910b055e2..2cf3a2f9f6 100644 --- a/src/domain/reconstructionVerificationSchemas.ts +++ b/src/domain/reconstructionVerificationSchemas.ts @@ -4,6 +4,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js"; +import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const unknownIdSchema = prefixedDigestSchema("unk"); @@ -26,14 +27,7 @@ const functionClaimSchema = z.object({ kind: z.literal("structural-function"), dimension: z.enum([ "overall", - "identity", - "pseudocode", - "assembly", - "comments", - "calls", - "references", - "strings_names", - "cfg", + ...functionComparisonDimensionNameSchema.options, ]), }); const artifactClaimSchema = z.object({ diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts index db82a04c8f..8eed65a485 100644 --- a/src/domain/staticRuntimeCorrelation.ts +++ b/src/domain/staticRuntimeCorrelation.ts @@ -14,7 +14,10 @@ import { } from "./comparisonSourceEvidence.js"; import { evidenceSchema, type Evidence } from "./evidence.js"; import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js"; -import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; +import { + functionComparisonDimensionNameSchema, + functionComparisonResultSchema, +} from "./functionComparisonSchemas.js"; import type { JsonValue } from "./jsonValue.js"; import { comparisonStatusSchema, @@ -24,16 +27,6 @@ import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const functionDimensionSchema = z.enum([ - "identity", - "pseudocode", - "assembly", - "comments", - "calls", - "references", - "strings_names", - "cfg", -]); const runtimeDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS); const expectedPatternSchema = z.enum([ "cochanged", @@ -54,7 +47,10 @@ const artifactSelectorSchema = z }) .strict(); const functionSelectorSchema = z - .object({ kind: z.literal("function"), dimension: functionDimensionSchema }) + .object({ + kind: z.literal("function"), + dimension: functionComparisonDimensionNameSchema, + }) .strict(); const staticSelectorSchema = z.discriminatedUnion("kind", [ artifactSelectorSchema, @@ -333,16 +329,11 @@ const selectStatic = ( "Function selector requires function comparison Evidence", ); const result = comparison.result; - const matches = result.dimensions.filter( + const match = result.dimensions.find( ({ dimension }) => dimension === selector.dimension, ); - const match = matches.at(0); - if (matches.length !== 1 || match === undefined) - throw callerInputError( - input, - path, - "Function selector must resolve exactly once", - ); + if (match === undefined) + throw evidenceIntegrityError("Function comparison dimension missing"); return { status: match.status, conclusion_kind: match.conclusion_kind, diff --git a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts new file mode 100644 index 0000000000..8615d92d7f --- /dev/null +++ b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts @@ -0,0 +1,262 @@ +import { expect, it, onTestFinished } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { + FUNCTION_COMPARISON_EVIDENCE, + INVESTIGATION_EXAMPLES, + PROCESS_CAPTURE_REFERENCE, + PROCESS_CAPTURE_RECONSTRUCTION, + PROCESS_COMPARISON_EVIDENCE, +} from "../../../src/contracts/investigationExamples.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + functionComparisonResultSchema, + type FunctionComparisonResult, +} from "../../../src/domain/functionComparisonSchemas.js"; +import { summarize } from "../../../src/domain/functionComparisonResults.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +const conflictingDimensions = (result: FunctionComparisonResult) => { + const changed = result.dimensions.find( + (dimension) => dimension.dimension === "pseudocode", + ); + if (changed?.status !== "changed") + throw new Error("Fixture requires a known pseudocode change"); + const dimensions = result.dimensions.map((dimension) => + dimension.dimension === "pseudocode" + ? { + ...changed, + status: "unchanged" as const, + conclusion_kind: "derived_relationship" as const, + } + : dimension.dimension === "cfg" + ? changed + : dimension, + ); + return { + ...result, + dimensions, + summary: summarize(dimensions), + changes: dimensions.filter(({ status }) => status !== "unchanged"), + }; +}; + +const unknownDimension = (result: FunctionComparisonResult) => { + const dimensions = result.dimensions.map((dimension) => + dimension.dimension === "cfg" + ? { + ...dimension, + status: "unknown" as const, + left_digest: null, + right_digest: null, + text_delta: null, + conclusion_kind: "unresolved_branch" as const, + } + : dimension, + ); + return { + ...result, + dimensions, + summary: summarize(dimensions), + changes: dimensions.filter(({ status }) => status !== "unchanged"), + }; +}; + +it.each([ + { + name: "canonical", + valid: true, + result: (value: FunctionComparisonResult) => value, + }, + { + name: "reordered canonical", + valid: true, + result: (value: FunctionComparisonResult) => ({ + ...value, + dimensions: [...value.dimensions].reverse(), + changes: [...value.changes].reverse(), + }), + }, + { + name: "unknown dimension with complete changes", + valid: true, + result: unknownDimension, + }, + { + name: "omitted unknown dimension with known changes retained", + valid: false, + result: (value: FunctionComparisonResult) => { + const unknown = unknownDimension(value); + return { + ...unknown, + changes: unknown.changes.filter(({ dimension }) => dimension !== "cfg"), + }; + }, + }, + { + name: "conflicting dimensions", + valid: false, + result: conflictingDimensions, + }, + { + name: "reversed conflicting dimensions", + valid: false, + result: (value: FunctionComparisonResult) => { + const malformed = conflictingDimensions(value); + return { + ...malformed, + dimensions: [...malformed.dimensions].reverse(), + changes: [...malformed.changes].reverse(), + }; + }, + }, + { + name: "incorrect summary counts with the correct total", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + summary: { + ...value.summary, + changed: value.summary.changed - 1, + unchanged: value.summary.unchanged + 1, + }, + }), + }, + { + name: "missing changes", + valid: false, + result: (value: FunctionComparisonResult) => ({ ...value, changes: [] }), + }, + { + name: "altered change provenance", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + changes: value.changes.map((dimension, index) => + index === 0 + ? { + ...dimension, + limitations: [ + ...dimension.limitations, + "Unattributed change description.", + ], + } + : dimension, + ), + }), + }, + { + name: "duplicate changes", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + changes: [...value.changes, ...value.changes], + }), + }, +])( + "validates $name function comparisons before investigation publication", + async (scenario) => { + const base = FUNCTION_COMPARISON_EVIDENCE; + const result = functionComparisonResultSchema.parse(base.normalized_result); + const comparison = createEvidence(undefined, base.provider, { + predicateType: base.predicate_type, + operation: base.operation, + parameters: base.parameters, + result: jsonValueSchema.parse(scenario.result(result)), + confidence: base.confidence, + authority: base.authority, + limitations: base.limitations, + evidenceLinks: base.evidence_links, + }); + let providerLaunches = 0; + const session = createTestBinarySession(() => { + providerLaunches += 1; + throw new Error( + "Retained comparison inspection must not launch a provider", + ); + }); + const harness = await createApplicationMcpHarness({ session }); + onTestFinished(harness.close); + for (const record of [ + FUNCTION_COMPARISON_EXAMPLE.left, + FUNCTION_COMPARISON_EXAMPLE.right, + PROCESS_CAPTURE_REFERENCE, + PROCESS_CAPTURE_RECONSTRUCTION, + PROCESS_COMPARISON_EVIDENCE, + comparison, + ]) + expect(session.recordEvidence(record).ok).toBe(true); + const recordCount = session.exportEvidenceBundle().records.length; + const correlation = INVESTIGATION_EXAMPLES.correlate_static_and_runtime; + const calls = [ + { + name: "find_changed_behavior", + arguments: { comparisons: [comparison] }, + }, + { + name: "correlate_static_and_runtime", + arguments: { + ...correlation, + static_comparisons: [comparison], + mappings: correlation.mappings.map((mapping) => ({ + ...mapping, + static: { + ...mapping.static, + comparison_evidence_id: comparison.evidence_id, + }, + })), + }, + }, + { + name: "verify_reconstruction", + arguments: { + specification: { + name: "Function comparison consistency", + claims: [ + { + kind: "structural-function", + claim_id: "pseudocode", + title: "Selected pseudocode remains equivalent", + comparison_evidence_id: comparison.evidence_id, + dimension: "pseudocode", + }, + ], + }, + }, + }, + ]; + const responses = []; + for (const call of calls) + responses.push({ call, response: await harness.client.callTool(call) }); + expect( + Object.fromEntries( + responses.map(({ call, response }) => [ + call.name, + response.isError === true, + ]), + ), + ).toEqual( + Object.fromEntries(calls.map(({ name }) => [name, !scenario.valid])), + ); + for (const { call, response } of responses) { + if (scenario.valid) + expect(parseEvidence(response.structuredContent).operation).toBe( + call.name, + ); + else + expect(parseMcpToolError(response)).toMatchObject({ + error: { + code: "evidence_integrity_mismatch", + category: "integrity_mismatch", + }, + }); + } + if (!scenario.valid) + expect(session.exportEvidenceBundle().records).toHaveLength(recordCount); + expect(session.evidenceById(comparison.evidence_id)).toEqual(comparison); + expect(providerLaunches).toBe(0); + }, +);