import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it, onTestFinished } from "vitest"; import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js"; import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE, FINALIZED_PROCESS_CAPTURE_EXAMPLE, } from "../../../src/domain/process/processCaptureExample.js"; import { capture, partialSpecification, processStarted, terminal, } from "../../../src/domain/process/processTraceComparison.fixture.js"; import { parseProcessCapture, type ProcessCapture, } from "../../../src/domain/process/processCaptureParsing.js"; import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js"; import { createEvidence, parseEvidence, type EvidenceProvider, } from "../../../src/domain/evidence.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; import { digestProcessCommitment } from "../../../src/domain/process/processScenario.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const captureEvidence = ( side: "left" | "right", comparisonContract: Readonly> = {}, provider: EvidenceProvider = PROCESS_PROVIDER, observation: ProcessCapture = parseProcessCapture( EMPTY_PROCESS_CAPTURE_EXAMPLE, ), ) => createEvidence(undefined, provider, { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: { side }, result: jsonValueSchema.parse({ ...observation, manifest: { ...observation.manifest, comparison_contract: comparisonContract, comparison_contract_sha256: digestProcessCommitment(comparisonContract), }, }), confidence: "observed", authority: "controlled-replay", }); const left = captureEvidence("left", { working_directory: "/work/reference" }); const right = captureEvidence("right", { working_directory: "/work/reconstruction", }); const connect = async ( session = createTestBinarySession(() => { throw new Error("Process comparison must not launch a provider"); }), ) => { const server = createServer({ kind: "session", session }); const client = new Client({ name: "process-comparison-test", version: "1" }); onTestFinished(async () => { await Promise.allSettled([client.close(), server.close(), session.close()]); }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); await server.connect(serverTransport); await client.connect(clientTransport); return client; }; const compareViaCli = async ( left: ReturnType, right: ReturnType, specification?: ProcessTraceSpecification, ) => { const root = await createTestTempDirectory("rea-process-provider-"); const leftPath = join(root, "left.json"); const rightPath = join(root, "right.json"); await Promise.all([ writeFile(leftPath, JSON.stringify(left)), writeFile(rightPath, JSON.stringify(right)), ]); const tracePath = join(root, "trace.json"); if (specification !== undefined) await writeFile(tracePath, JSON.stringify(specification)); return compareProcessEvidenceFiles( leftPath, rightPath, specification === undefined ? undefined : tracePath, ); }; it.each( (["left", "right"] as const).flatMap((missingWorker) => (["left", "right", "both", "neither"] as const).map((omittedRendered) => ({ missingWorker, omittedRendered, })), ), )( "preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output", async ({ missingWorker, omittedRendered }) => { const observation = (side: "left" | "right") => { const hasWorker = side !== missingWorker; const complete = capture({ frames: [terminal], process_samples: hasWorker ? [processStarted] : [], event_journal: [ { collection: "frames", index: 0 }, ...(hasWorker ? [{ collection: "process_samples" as const, index: 0 }] : []), ], }); const omitted = omittedRendered === "both" || omittedRendered === side; return captureEvidence( side, {}, PROCESS_PROVIDER, omitted ? parseProcessCapture({ ...complete, truncated: true, truncation_details: { ...complete.truncation_details, rendered_terminal: { budget_bytes: 5, observed_bytes: 0, retained_bytes: 0, observed_frames: 1, retained_frames: 0, unknown_size_frames: 1, }, }, }) : complete, ); }; const left = observation("left"); const right = observation("right"); const terminalStatus = omittedRendered === "neither" ? "unchanged" : "unknown"; const plain = parseEvidence(await compareViaCli(left, right)); expect(plain.normalized_result).toMatchObject({ terminal: terminalStatus, process: missingWorker === "left" ? "added" : "removed", status: "changed", }); const specification = partialSpecification(); const cli = parseEvidence(await compareViaCli(left, right, specification)); const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left, right, trace_spec: specification }, }); expect(response.isError).not.toBe(true); expect(parseEvidence(response.structuredContent)).toEqual(cli); expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]); expect(cli.normalized_result).toMatchObject({ terminal: terminalStatus, process: "changed", status: "changed", trace: { verdict: "different", diagnostic: { kind: "cardinality", side: missingWorker, event_ids: ["worker"], locations: [], }, }, }); }, ); it.each( (["left", "right"] as const).flatMap((side) => [ { id: "unrelated-provider" }, { version: "1" }, { name: "unrelated provider" }, ].map((change) => ({ side, change })), ), )( "rejects an authenticated $side capture with provider $change", async ({ side, change }) => { const wrongProvider = captureEvidence( side, {}, { ...PROCESS_PROVIDER, ...change }, ); expect(parseEvidence(wrongProvider)).toEqual(wrongProvider); const left = side === "left" ? wrongProvider : captureEvidence("left"); const right = side === "right" ? wrongProvider : captureEvidence("right"); const issue = { path: [side, "provider"], reason: "invalid_value", message: expect.stringContaining("process capture provider"), }; expect(await compareViaCli(left, right)).toMatchObject({ category: "invalid_input", details: { operation: "compare_process_captures", issues: [issue] }, }); const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left, right }, }); expect(response.isError).toBe(true); expect(parseMcpToolError(response)).toMatchObject({ error: { code: "invalid_request", details: { issues: [issue] } }, }); await client.ping(); }, ); it.each([PROCESS_PROVIDER.name, "REA deterministic process harness"])( "preserves authenticated CLI/MCP comparison Evidence for provider name %s", async (name) => { const provider = { ...PROCESS_PROVIDER, name }; const left = captureEvidence("left", {}, provider); const right = captureEvidence("right", {}, provider); const cli = parseEvidence(await compareViaCli(left, right)); const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left, right }, }); expect(response.isError).not.toBe(true); expect(parseEvidence(response.structuredContent)).toEqual(cli); const text = response.content.find((block) => block.type === "text"); expect(text?.type).toBe("text"); if (text?.type !== "text") throw new Error("Missing comparison text"); expect(JSON.parse(text.text)).toEqual(cli); expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]); }, ); it.each(["left", "right"] as const)( "authenticates the %s Evidence before provider admission", async (side) => { const mismatched = captureEvidence( side, {}, { ...PROCESS_PROVIDER, id: "unrelated-provider" }, ); const tampered = { ...mismatched, parameters: { side: "tampered" } }; const left = side === "left" ? tampered : captureEvidence("left"); const right = side === "right" ? tampered : captureEvidence("right"); expect(await compareViaCli(left, right)).toMatchObject({ category: "invalid_input", details: { issues: [ { path: ["evidence_id"], message: expect.stringContaining( "Evidence semantic identifier does not match its record", ), }, ], }, }); const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left, right }, }); expect(parseMcpToolError(response)).toMatchObject({ error: { code: "evidence_integrity_mismatch" }, }); }, ); it("names differing comparison contract fields through MCP and the CLI", async () => { const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left, right }, }); expect(response.isError).toBe(true); const issue = { path: ["right"], reason: "invalid_value", message: expect.stringContaining( "these scenario fields differ: working_directory", ), expected: ["working_directory"], }; expect(parseMcpToolError(response)).toMatchObject({ error: { code: "invalid_request", details: { issues: [issue] } }, }); const root = await createTestTempDirectory("rea-process-contract-"); const leftPath = join(root, "left.json"); const rightPath = join(root, "right.json"); await writeFile(leftPath, JSON.stringify(left)); await writeFile(rightPath, JSON.stringify(right)); expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({ error: "Process command failed", code: "invalid_request", details: { operation: "compare_process_captures", issues: [issue] }, }); }); it("names the stale capture that exceeds max_capture_age_ms", async () => { const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left: captureEvidence("left"), right: captureEvidence("right"), max_capture_age_ms: 1000, }, }); expect(response.isError).toBe(true); expect(parseMcpToolError(response)).toMatchObject({ error: { code: "invalid_request", details: { issues: [ { path: ["max_capture_age_ms"], reason: "out_of_range", message: expect.stringContaining( `left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`, ), }, ], }, }, }); }); it("retains a selected max_capture_age_ms requirement in comparison Evidence parameters", async () => { const session = createTestBinarySession(() => { throw new Error("Process comparison must not launch a provider"); }); const client = await connect(session); const unconstrained = await client.callTool({ name: "compare_process_captures", arguments: { left: captureEvidence("left"), right: captureEvidence("right"), }, }); const selected = await client.callTool({ name: "compare_process_captures", arguments: { left: captureEvidence("left"), right: captureEvidence("right"), max_capture_age_ms: Number.MAX_SAFE_INTEGER, }, }); expect(unconstrained.isError).not.toBe(true); expect(selected.isError).not.toBe(true); const before = parseEvidence(unconstrained.structuredContent); const after = parseEvidence(selected.structuredContent); expect(before.parameters).not.toHaveProperty("max_capture_age_ms"); expect(after.parameters).toHaveProperty( "max_capture_age_ms", Number.MAX_SAFE_INTEGER, ); expect(after.evidence_id).not.toBe(before.evidence_id); expect(session.evidenceById(after.evidence_id)?.parameters).toHaveProperty( "max_capture_age_ms", Number.MAX_SAFE_INTEGER, ); }); it("reports CLI trace specification issues like the MCP trace_spec input", async () => { const root = await createTestTempDirectory("rea-process-trace-spec-"); const capturePath = join(root, "capture.json"); const specPath = join(root, "spec.json"); await writeFile(capturePath, JSON.stringify(captureEvidence("left"))); await writeFile(specPath, JSON.stringify({ language: {} })); expect( await compareProcessEvidenceFiles(capturePath, capturePath, specPath), ).toMatchObject({ code: "invalid_request", details: { operation: "compare_process_captures", issues: expect.arrayContaining([ expect.objectContaining({ path: ["trace_spec", "events"] }), ]), }, }); }); it("rejects a deep trace cycle as an input error through CLI and MCP", async () => { const events = Array.from({ length: 6000 }, (_, index) => ({ id: `e${String(index)}`, source: "lifecycle", exact: { marker: index }, })); const trace = { events, language: { kind: "partial_order", happens_before: events.map((event, index) => ({ before: event.id, after: events[(index + 1) % events.length]!.id, })), not_before: events.map((event, index) => ({ event: event.id, anchor: events[(index + 1) % events.length]!.id, })), unordered_groups: [{ events: events.map(({ id }) => id) }], }, }; const issue = { path: ["trace_spec", "language"], message: "happens-before graph must be acyclic", }; const capture = captureEvidence("left"); const root = await createTestTempDirectory("rea-process-trace-cycle-"); const capturePath = join(root, "capture.json"); const specPath = join(root, "spec.json"); await writeFile(capturePath, JSON.stringify(capture)); await writeFile(specPath, JSON.stringify(trace)); expect( await compareProcessEvidenceFiles(capturePath, capturePath, specPath), ).toMatchObject({ code: "invalid_request", details: { issues: [issue] }, }); const client = await connect(); const response = await client.callTool({ name: "compare_process_captures", arguments: { left: capture, right: capture, trace_spec: trace }, }); expect(response.isError).toBe(true); expect(response.content).toEqual([ { type: "text", text: expect.stringContaining(issue.message) }, ]); await client.ping(); }); it("names the side and constraint when Evidence is not a usable capture", async () => { const client = await connect(); const capture = captureEvidence("right"); const unrelated = createEvidence(undefined, PROCESS_PROVIDER, { operation: "other", parameters: {}, result: {}, }); const wrongKind = await client.callTool({ name: "compare_process_captures", arguments: { left: unrelated, right: capture }, }); expect(parseMcpToolError(wrongKind)).toMatchObject({ error: { code: "invalid_request", details: { issues: [ { path: ["left"], reason: "invalid_value", message: expect.stringContaining( "the left Evidence has operation other", ), }, ], }, }, }); const tampered = await client.callTool({ name: "compare_process_captures", arguments: { left: capture, right: { ...capture, parameters: { side: "changed" } }, }, }); expect(parseMcpToolError(tampered)).toMatchObject({ error: { code: "evidence_integrity_mismatch", message: expect.stringContaining( "The right Evidence failed validation (at evidence_id: Evidence semantic identifier does not match its record)", ), }, }); const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, result: { manifest: {} }, }); const malformed = await client.callTool({ name: "compare_process_captures", arguments: { left: capture, right: invalidResult }, }); expect(parseMcpToolError(malformed)).toMatchObject({ error: { code: "evidence_integrity_mismatch", message: expect.stringContaining( "The right Evidence has an invalid process capture result (at ", ), }, }); const root = await createTestTempDirectory("rea-process-malformed-"); const leftPath = join(root, "left.json"); const rightPath = join(root, "right.json"); await writeFile(leftPath, JSON.stringify(capture)); await writeFile(rightPath, JSON.stringify(invalidResult)); expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({ category: "invalid_input", message: expect.stringContaining("Capture evidence is malformed (at "), }); }); it("keeps structural capture issues alongside semantic ones through MCP and the CLI", async () => { const client = await connect(); const capture = captureEvidence("left"); const { manifest: { completed_at: _completedAt, ...manifest }, exit, ...rest } = FINALIZED_PROCESS_CAPTURE_EXAMPLE; const mixed = createEvidence(undefined, PROCESS_PROVIDER, { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: {}, result: jsonValueSchema.parse({ ...rest, manifest, exit: { ...exit, finalization: { ...exit.finalization, signals: [ { ...exit.finalization.signals[0], reason: "not allowed here" }, ], }, }, }), }); const described = "(at manifest.completed_at: Invalid input: expected string, received undefined (1 more issues))"; const response = await client.callTool({ name: "compare_process_captures", arguments: { left: capture, right: mixed }, }); expect(parseMcpToolError(response)).toMatchObject({ error: { code: "evidence_integrity_mismatch", message: expect.stringContaining( `The right Evidence has an invalid process capture result ${described}`, ), }, }); expect(await compareViaCli(capture, mixed)).toMatchObject({ code: "invalid_request", category: "invalid_input", message: expect.stringContaining( `Capture evidence is malformed ${described}`, ), }); });