diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts index 75957a7b70..7c4e1dfbb5 100644 --- a/src/domain/functionComparisonSchemas.ts +++ b/src/domain/functionComparisonSchemas.ts @@ -1,5 +1,6 @@ import { z } from "zod"; -import { overallStatus } from "./functionComparisonResults.js"; +import { overallStatus, summarize } from "./functionComparisonResults.js"; +import { canonicalJson } from "./comparisonSemantics.js"; import { evidenceInputSchema } from "./evidenceReference.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -91,11 +92,60 @@ export const functionComparisonResultSchema = z changed: z.int().min(0), unknown: z.int().min(0), }), - dimensions: z.array(functionDimensionSchema).length(8), + dimensions: z + .array(functionDimensionSchema) + .length(functionComparisonDimensionNameSchema.options.length), changes: z.array(functionDimensionSchema), limitations: z.array(z.string()), }) .superRefine((result, context) => { + const dimensions = new Map(); + for (const [index, dimension] of result.dimensions.entries()) { + if (dimensions.has(dimension.dimension)) + context.addIssue({ + code: "custom", + path: ["dimensions", index, "dimension"], + message: "Function comparison dimensions must occur exactly once.", + }); + dimensions.set(dimension.dimension, dimension); + } + const expectedSummary = summarize(result.dimensions); + for (const status of comparisonStatusSchema.options) + if (result.summary[status] !== expectedSummary[status]) + context.addIssue({ + code: "custom", + path: ["summary", status], + message: "Function comparison summary must count its dimensions.", + }); + const changes = new Set(); + for (const [index, change] of result.changes.entries()) { + const dimension = dimensions.get(change.dimension); + if ( + changes.has(change.dimension) || + dimension === undefined || + change.status === "unchanged" || + canonicalJson(change) !== canonicalJson(dimension) + ) + context.addIssue({ + code: "custom", + path: ["changes", index], + message: + "Function comparison changes must cite each changed or unknown dimension exactly as recorded.", + }); + changes.add(change.dimension); + } + if ( + result.dimensions.some( + ({ dimension, status }) => + status !== "unchanged" && !changes.has(dimension), + ) + ) + context.addIssue({ + code: "custom", + path: ["changes"], + message: + "Function comparison changes must include every changed or unknown dimension.", + }); if ( result.status !== overallStatus(result.dimensions, result.function_match.status) diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts index eadb957c40..c053730812 100644 --- a/src/domain/reconstructionVerification.functionSources.test.ts +++ b/src/domain/reconstructionVerification.functionSources.test.ts @@ -120,23 +120,27 @@ const comparison = ( target?: EvidenceSubjectTarget; } = {}, resultInput = baselineResult, -): Evidence => - createEvidence(identity.target, identity.provider ?? provider, { +): Evidence => { + const links = sources.map(({ evidence_id: id }) => id); + const dimensions = resultInput.dimensions.map((dimension) => ({ + ...dimension, + evidence_links: links, + })); + return createEvidence(identity.target, identity.provider ?? provider, { predicateType: "rea.function-comparison", operation: "compare_functions", parameters, result: jsonValueSchema.parse({ ...resultInput, - dimensions: resultInput.dimensions.map((dimension) => ({ - ...dimension, - evidence_links: sources.map(({ evidence_id: id }) => id), - })), + dimensions, + changes: dimensions.filter(({ status }) => status !== "unchanged"), }), confidence: "derived", authority: "analyst-inference", - evidenceLinks: sources.map(({ evidence_id: id }) => id), + evidenceLinks: links, ...observation, }); +}; const specification = (evidence: Evidence, dimension = "pseudocode") => ({ name: "Function reconstruction", diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.ts index 1910b055e2..2cf3a2f9f6 100644 --- a/src/domain/reconstructionVerificationSchemas.ts +++ b/src/domain/reconstructionVerificationSchemas.ts @@ -4,6 +4,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js"; +import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const unknownIdSchema = prefixedDigestSchema("unk"); @@ -26,14 +27,7 @@ const functionClaimSchema = z.object({ kind: z.literal("structural-function"), dimension: z.enum([ "overall", - "identity", - "pseudocode", - "assembly", - "comments", - "calls", - "references", - "strings_names", - "cfg", + ...functionComparisonDimensionNameSchema.options, ]), }); const artifactClaimSchema = z.object({ diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts index db82a04c8f..8eed65a485 100644 --- a/src/domain/staticRuntimeCorrelation.ts +++ b/src/domain/staticRuntimeCorrelation.ts @@ -14,7 +14,10 @@ import { } from "./comparisonSourceEvidence.js"; import { evidenceSchema, type Evidence } from "./evidence.js"; import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js"; -import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; +import { + functionComparisonDimensionNameSchema, + functionComparisonResultSchema, +} from "./functionComparisonSchemas.js"; import type { JsonValue } from "./jsonValue.js"; import { comparisonStatusSchema, @@ -24,16 +27,6 @@ import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); -const functionDimensionSchema = z.enum([ - "identity", - "pseudocode", - "assembly", - "comments", - "calls", - "references", - "strings_names", - "cfg", -]); const runtimeDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS); const expectedPatternSchema = z.enum([ "cochanged", @@ -54,7 +47,10 @@ const artifactSelectorSchema = z }) .strict(); const functionSelectorSchema = z - .object({ kind: z.literal("function"), dimension: functionDimensionSchema }) + .object({ + kind: z.literal("function"), + dimension: functionComparisonDimensionNameSchema, + }) .strict(); const staticSelectorSchema = z.discriminatedUnion("kind", [ artifactSelectorSchema, @@ -333,16 +329,11 @@ const selectStatic = ( "Function selector requires function comparison Evidence", ); const result = comparison.result; - const matches = result.dimensions.filter( + const match = result.dimensions.find( ({ dimension }) => dimension === selector.dimension, ); - const match = matches.at(0); - if (matches.length !== 1 || match === undefined) - throw callerInputError( - input, - path, - "Function selector must resolve exactly once", - ); + if (match === undefined) + throw evidenceIntegrityError("Function comparison dimension missing"); return { status: match.status, conclusion_kind: match.conclusion_kind, diff --git a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts new file mode 100644 index 0000000000..8615d92d7f --- /dev/null +++ b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts @@ -0,0 +1,262 @@ +import { expect, it, onTestFinished } from "vitest"; + +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { + FUNCTION_COMPARISON_EVIDENCE, + INVESTIGATION_EXAMPLES, + PROCESS_CAPTURE_REFERENCE, + PROCESS_CAPTURE_RECONSTRUCTION, + PROCESS_COMPARISON_EVIDENCE, +} from "../../../src/contracts/investigationExamples.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + functionComparisonResultSchema, + type FunctionComparisonResult, +} from "../../../src/domain/functionComparisonSchemas.js"; +import { summarize } from "../../../src/domain/functionComparisonResults.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +const conflictingDimensions = (result: FunctionComparisonResult) => { + const changed = result.dimensions.find( + (dimension) => dimension.dimension === "pseudocode", + ); + if (changed?.status !== "changed") + throw new Error("Fixture requires a known pseudocode change"); + const dimensions = result.dimensions.map((dimension) => + dimension.dimension === "pseudocode" + ? { + ...changed, + status: "unchanged" as const, + conclusion_kind: "derived_relationship" as const, + } + : dimension.dimension === "cfg" + ? changed + : dimension, + ); + return { + ...result, + dimensions, + summary: summarize(dimensions), + changes: dimensions.filter(({ status }) => status !== "unchanged"), + }; +}; + +const unknownDimension = (result: FunctionComparisonResult) => { + const dimensions = result.dimensions.map((dimension) => + dimension.dimension === "cfg" + ? { + ...dimension, + status: "unknown" as const, + left_digest: null, + right_digest: null, + text_delta: null, + conclusion_kind: "unresolved_branch" as const, + } + : dimension, + ); + return { + ...result, + dimensions, + summary: summarize(dimensions), + changes: dimensions.filter(({ status }) => status !== "unchanged"), + }; +}; + +it.each([ + { + name: "canonical", + valid: true, + result: (value: FunctionComparisonResult) => value, + }, + { + name: "reordered canonical", + valid: true, + result: (value: FunctionComparisonResult) => ({ + ...value, + dimensions: [...value.dimensions].reverse(), + changes: [...value.changes].reverse(), + }), + }, + { + name: "unknown dimension with complete changes", + valid: true, + result: unknownDimension, + }, + { + name: "omitted unknown dimension with known changes retained", + valid: false, + result: (value: FunctionComparisonResult) => { + const unknown = unknownDimension(value); + return { + ...unknown, + changes: unknown.changes.filter(({ dimension }) => dimension !== "cfg"), + }; + }, + }, + { + name: "conflicting dimensions", + valid: false, + result: conflictingDimensions, + }, + { + name: "reversed conflicting dimensions", + valid: false, + result: (value: FunctionComparisonResult) => { + const malformed = conflictingDimensions(value); + return { + ...malformed, + dimensions: [...malformed.dimensions].reverse(), + changes: [...malformed.changes].reverse(), + }; + }, + }, + { + name: "incorrect summary counts with the correct total", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + summary: { + ...value.summary, + changed: value.summary.changed - 1, + unchanged: value.summary.unchanged + 1, + }, + }), + }, + { + name: "missing changes", + valid: false, + result: (value: FunctionComparisonResult) => ({ ...value, changes: [] }), + }, + { + name: "altered change provenance", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + changes: value.changes.map((dimension, index) => + index === 0 + ? { + ...dimension, + limitations: [ + ...dimension.limitations, + "Unattributed change description.", + ], + } + : dimension, + ), + }), + }, + { + name: "duplicate changes", + valid: false, + result: (value: FunctionComparisonResult) => ({ + ...value, + changes: [...value.changes, ...value.changes], + }), + }, +])( + "validates $name function comparisons before investigation publication", + async (scenario) => { + const base = FUNCTION_COMPARISON_EVIDENCE; + const result = functionComparisonResultSchema.parse(base.normalized_result); + const comparison = createEvidence(undefined, base.provider, { + predicateType: base.predicate_type, + operation: base.operation, + parameters: base.parameters, + result: jsonValueSchema.parse(scenario.result(result)), + confidence: base.confidence, + authority: base.authority, + limitations: base.limitations, + evidenceLinks: base.evidence_links, + }); + let providerLaunches = 0; + const session = createTestBinarySession(() => { + providerLaunches += 1; + throw new Error( + "Retained comparison inspection must not launch a provider", + ); + }); + const harness = await createApplicationMcpHarness({ session }); + onTestFinished(harness.close); + for (const record of [ + FUNCTION_COMPARISON_EXAMPLE.left, + FUNCTION_COMPARISON_EXAMPLE.right, + PROCESS_CAPTURE_REFERENCE, + PROCESS_CAPTURE_RECONSTRUCTION, + PROCESS_COMPARISON_EVIDENCE, + comparison, + ]) + expect(session.recordEvidence(record).ok).toBe(true); + const recordCount = session.exportEvidenceBundle().records.length; + const correlation = INVESTIGATION_EXAMPLES.correlate_static_and_runtime; + const calls = [ + { + name: "find_changed_behavior", + arguments: { comparisons: [comparison] }, + }, + { + name: "correlate_static_and_runtime", + arguments: { + ...correlation, + static_comparisons: [comparison], + mappings: correlation.mappings.map((mapping) => ({ + ...mapping, + static: { + ...mapping.static, + comparison_evidence_id: comparison.evidence_id, + }, + })), + }, + }, + { + name: "verify_reconstruction", + arguments: { + specification: { + name: "Function comparison consistency", + claims: [ + { + kind: "structural-function", + claim_id: "pseudocode", + title: "Selected pseudocode remains equivalent", + comparison_evidence_id: comparison.evidence_id, + dimension: "pseudocode", + }, + ], + }, + }, + }, + ]; + const responses = []; + for (const call of calls) + responses.push({ call, response: await harness.client.callTool(call) }); + expect( + Object.fromEntries( + responses.map(({ call, response }) => [ + call.name, + response.isError === true, + ]), + ), + ).toEqual( + Object.fromEntries(calls.map(({ name }) => [name, !scenario.valid])), + ); + for (const { call, response } of responses) { + if (scenario.valid) + expect(parseEvidence(response.structuredContent).operation).toBe( + call.name, + ); + else + expect(parseMcpToolError(response)).toMatchObject({ + error: { + code: "evidence_integrity_mismatch", + category: "integrity_mismatch", + }, + }); + } + if (!scenario.valid) + expect(session.exportEvidenceBundle().records).toHaveLength(recordCount); + expect(session.evidenceById(comparison.evidence_id)).toEqual(comparison); + expect(providerLaunches).toBe(0); + }, +);