diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index 68fd84953a..700f202b62 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -18,8 +18,6 @@ import { import { functionComparisonResultSchema } from "./functionComparisonSchemas.js"; import { comparisonStatusSchema, - deriveProcessComparisonStatus, - PROCESS_COMPARISON_DIMENSIONS, processCaptureComparisonSchema, } from "./process/processComparison.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; @@ -209,13 +207,6 @@ const parseResult = ( const result = processCaptureComparisonSchema.parse( evidence.normalized_result, ); - const expected = deriveProcessComparisonStatus( - PROCESS_COMPARISON_DIMENSIONS.map((dimension) => result[dimension]), - ); - if (result.status !== expected) - throw new TypeError( - "Process comparison status contradicts its dimensions", - ); return { operation, evidence, result }; } if (operation === "compare_functions") { diff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.ts index 0bc69335d3..7c7cbd590b 100644 --- a/src/domain/process/processCapture.comparison.test.ts +++ b/src/domain/process/processCapture.comparison.test.ts @@ -1,5 +1,11 @@ import { expect, it } from "vitest"; +import { + deriveProcessComparisonStatus, + PROCESS_COMPARISON_DIMENSIONS, + processCaptureComparisonSchema, +} from "./processComparison.js"; + import { accountFullyObservedProcessCapture, compareUnverifiedProcessCaptures as compareProcessCaptures, @@ -17,6 +23,58 @@ const compareFullyObserved = ( options, ); +it.each(PROCESS_COMPARISON_DIMENSIONS)( + "requires a contradiction aggregate for a contradictory %s dimension", + (dimension) => { + const baseline = compareFullyObserved(emptyCapture(), emptyCapture()); + const result = { ...baseline, [dimension]: "contradiction" as const }; + expect( + deriveProcessComparisonStatus( + PROCESS_COMPARISON_DIMENSIONS.map((name) => result[name]), + ), + ).toBe("contradiction"); + expect(processCaptureComparisonSchema.safeParse(result).success).toBe( + false, + ); + expect( + processCaptureComparisonSchema.parse({ + ...result, + status: "contradiction", + }), + ).toMatchObject({ status: "contradiction", [dimension]: "contradiction" }); + }, +); + +it.each([ + ["unchanged", "contradiction"], + ["added", "contradiction"], + ["removed", "contradiction"], + ["changed", "contradiction"], + ["unknown", "contradiction"], + ["contradiction", "contradiction"], + ["truncated", "truncated"], +] as const)( + "combines contradiction with %s as %s independently of dimension order", + (other, expected) => { + for (const [terminal, interaction] of [ + ["contradiction", other], + [other, "contradiction"], + ] as const) { + expect(deriveProcessComparisonStatus([terminal, interaction])).toBe( + expected, + ); + expect( + processCaptureComparisonSchema.parse({ + ...compareFullyObserved(emptyCapture(), emptyCapture()), + terminal, + interaction, + status: expected, + }), + ).toMatchObject({ status: expected }); + } + }, +); + it("compares normalized terminal facts while retaining different source text for audit", () => { const capture = emptyCapture(); const frame = { sequence: 0, at_ms: 0, data: "listen: score=20" }; diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index 0b3434a037..69dc1ac566 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -49,6 +49,7 @@ export const deriveProcessComparisonStatus = ( dimensions: readonly ComparisonStatus[], ): ComparisonStatus => { if (dimensions.includes("truncated")) return "truncated"; + if (dimensions.includes("contradiction")) return "contradiction"; if ( dimensions.some((status) => ["changed", "added", "removed"].includes(status), diff --git a/tests/boundary/mcp/processComparisonVerificationMcp.test.ts b/tests/boundary/mcp/processComparisonVerificationMcp.test.ts new file mode 100644 index 0000000000..618bc33ae1 --- /dev/null +++ b/tests/boundary/mcp/processComparisonVerificationMcp.test.ts @@ -0,0 +1,143 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, onTestFinished } from "vitest"; + +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; +import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; +import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; +import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; +import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("rejects an inconsistent imported process comparison while retaining its Evidence", async () => { + const dimension = "terminal" as const; + const source = (side: string) => + createEvidence(undefined, PROCESS_PROVIDER, { + predicateType: "rea.process-capture", + operation: "capture_process_scenario", + parameters: { side }, + result: EMPTY_PROCESS_CAPTURE_EXAMPLE, + confidence: "observed", + authority: "controlled-replay", + environment: { + id: "fixture", + platform: "fixture", + architecture: "fixture", + isolation: "process", + }, + }); + const left = source("left"); + const right = source("right"); + const nativeResult = compareProcessCaptures( + parseProcessCapture(left.normalized_result), + parseProcessCapture(right.normalized_result), + ); + expect(nativeResult.status).toBe("unchanged"); + const comparison = (status: "unchanged" | "contradiction") => + createEvidence(undefined, PROCESS_PROVIDER, { + predicateType: "rea.process-comparison", + operation: "compare_process_captures", + parameters: { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + }, + result: jsonValueSchema.parse({ + ...nativeResult, + [dimension]: "contradiction", + status, + }), + confidence: "derived", + authority: "analyst-inference", + evidenceLinks: [left.evidence_id, right.evidence_id], + }); + const inconsistent = comparison("unchanged"); + const consistent = comparison("contradiction"); + const root = await createTestTempDirectory("rea-process-verification-"); + const path = join(root, "bundle.json"); + await writeFile( + path, + JSON.stringify( + createEvidenceBundle([left, right, inconsistent, consistent]), + ), + ); + let providerLaunches = 0; + const session = createTestBinarySession(() => { + providerLaunches += 1; + throw new Error("Imported process verification must not launch a provider"); + }); + const harness = await createApplicationMcpHarness({ session }); + onTestFinished(harness.close); + const imported = await harness.client.callTool({ + name: "import_evidence_bundle", + arguments: { path }, + }); + expect(imported.isError).not.toBe(true); + const verify = (evidenceId: string, selected: "overall" | typeof dimension) => + harness.client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Process comparison consistency", + claims: [ + { + kind: "behavioral", + claim_id: selected, + title: "Process replay agrees", + comparison_evidence_id: evidenceId, + dimension: selected, + }, + ], + }, + }, + }); + for (const selected of ["overall", dimension] as const) { + const rejected = await verify(inconsistent.evidence_id, selected); + expect(parseMcpToolError(rejected).error).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + reason: "invalid_value", + message: expect.stringContaining( + "Process comparison status contradicts its dimensions", + ), + }, + ], + }, + }); + const failed = await verify(consistent.evidence_id, selected); + expect(failed.isError).not.toBe(true); + expect( + reconstructionVerificationResultSchema.parse( + parseEvidence(failed.structuredContent).normalized_result, + ), + ).toMatchObject({ + status: "fail", + claims: { items: [{ observed_status: "contradiction" }] }, + }); + } + for (const record of [left, right, inconsistent, consistent]) + expect(session.evidenceById(record.evidence_id)).toEqual(record); + const recomputed = await harness.client.callTool({ + name: "compare_process_captures", + arguments: { left, right }, + }); + expect(recomputed.isError).not.toBe(true); + const control = parseEvidence(recomputed.structuredContent); + const verified = await verify(control.evidence_id, "overall"); + expect(verified.isError).not.toBe(true); + expect( + reconstructionVerificationResultSchema.parse( + parseEvidence(verified.structuredContent).normalized_result, + ), + ).toMatchObject({ status: "pass" }); + expect(providerLaunches).toBe(0); + await harness.client.ping(); +});