From 63ea5168327fbb951a1ff6ed47c28d9c7d764855 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:20:35 -0500 Subject: [PATCH] test(parsers): compare adapters with admitted provider results (#12203) --- .../mcp/analysisEvidenceAdapterParity.test.ts | 128 ++++++++++++++---- 1 file changed, 104 insertions(+), 24 deletions(-) diff --git a/tests/boundary/mcp/analysisEvidenceAdapterParity.test.ts b/tests/boundary/mcp/analysisEvidenceAdapterParity.test.ts index cbcba72b3d..1495ebcc1b 100644 --- a/tests/boundary/mcp/analysisEvidenceAdapterParity.test.ts +++ b/tests/boundary/mcp/analysisEvidenceAdapterParity.test.ts @@ -13,6 +13,7 @@ import { import { createCliShell } from "../../../src/cli/cliShell.js"; import type { ContractCliDescription } from "../../../src/cli/contractCliDescription.js"; import { registeredContractDescriptions } from "../../support/cli/commandCli.js"; +import { createArtifactBinarySession } from "../../../src/composition/binary.js"; import { bindDirectAnalysis } from "../../../src/composition/directAnalysis.js"; import { parseConfig } from "../../../src/config/parseConfig.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../../../src/contracts/artifactToolContracts.js"; @@ -25,6 +26,7 @@ import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { ok } from "../../../src/domain/result.js"; import { createAnalysisExecution } from "../../../src/ports/AnalysisProvider.js"; import { createServer } from "../../../src/server/createServer.js"; +import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js"; import { createTestProviderRouter } from "../../fixtures/binarySession.js"; import { minimalContractInput, @@ -33,20 +35,21 @@ import { import { minimalMachoImage } from "../../fixtures/nativeHeaders.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { serveInProcessCli } from "../../support/cli/serveInProcessCli.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; /** * One call, two adapters: every analysis operation the CLI exposes runs once * through the CLI command tree and once through the MCP server, over the same - * simulated provider and target. Both must deliver the same Evidence, or the + * provider and target. Both must deliver the same Evidence, or the * same failure. * - * KNOWN_DIFFERENCES records how main differs today. The W2 invoke path - * removes them; when a change removes one, delete its entry. An entry never - * grows. Both adapters must retain the exact caller-selected spelling. + * KNOWN_DIFFERENCES records the exact Evidence and failure fields where the + * adapters still differ. Both adapters must retain the caller’s exact spelling. * - * Simulated providers return the smallest result each contract admits, so - * this checks adapter composition, not real provider behavior. + * Simulated providers use authored results where the JSON Schema cannot + * express the complete runtime contract. Archive cases use the production + * artifact reader over a small ZIP fixture. */ /** JSON paths, under `evidence.` or `failure.`, where CLI and MCP differ. */ @@ -64,6 +67,9 @@ const KNOWN_DIFFERENCES: Readonly> = { "inspect_keyed_archive", "inspect_asset_catalog", "trace_dylib_resolution", + "observe_native_ui", + "capture_native_ui_scenario", + "extract_artifact", ].map((name) => [ name, ["evidence.evidence_id", "evidence.parameters.input_path"], @@ -83,16 +89,15 @@ const KNOWN_DIFFERENCES: Readonly> = { ], ]), ), + // The MCP inspection records the delivery detail in Evidence parameters. + inspect_artifact: [ + "evidence.evidence_id", + "evidence.parameters.detail", + "evidence.parameters.input_path", + ], // `DirectAnalysis` shapes CLI failures in the application layer and adds // an `error: "Analysis failed"` marker MCP does not send. trace_native_ui_action: ["failure.error"], - // MCP checks the result against the contract output schema; the CLI - // delivers Evidence whose result that schema refuses. - read_bytes: ["outcome"], - observe_native_ui: ["outcome"], - capture_native_ui_scenario: ["outcome"], - inspect_artifact: ["outcome"], - extract_artifact: ["outcome"], }; /** Inputs a one-shot CLI call needs that MCP takes from session state. */ @@ -105,8 +110,61 @@ const INPUT_OVERRIDES: Readonly< const PROVIDER = { id: "fixture", name: "Fixture", version: "1" } as const; +/** Valid replies for contracts with relationships absent from JSON Schema. */ +const nativeUiResult = { + target_sha256: "a".repeat(64), + initial: { + window: { + pid: 123, + window_id: 456, + executable: "/fixture", + launch_time: 0, + title: "Fixture", + }, + nodes: [], + truncated: false, + screenshot: null, + gaps: [], + }, + steps: [], + restore: "leave-as-is", + limitations: [], +} satisfies JsonValue; + +const authoredResults = new Map([ + [ + "read_bytes", + { + address: "0x1000", + requested_bytes: 16, + returned_bytes: 1, + bytes_hex: "ab", + complete: false, + }, + ], + ["observe_native_ui", nativeUiResult], + [ + "capture_native_ui_scenario", + { + ...nativeUiResult, + steps: [ + { + index: 0, + kind: "wait", + before: nativeUiResult.initial, + after: nativeUiResult.initial, + outcome: "completed", + reason: null, + }, + ], + }, + ], +]); + const results = new Map(); const resultFor = (operation: string): JsonValue | undefined => { + const authored = authoredResults.get(operation); + if (authored !== undefined) return authored; if (!results.has(operation)) { const contract = TOOL_CONTRACTS.find(({ name }) => name === operation); results.set( @@ -149,10 +207,10 @@ const registrations = await Promise.all( ); /** A fresh CLI command tree over the simulated provider. */ -const createCli = () => +const createCli = (runWith = dependencies) => createCliShell({ selectedEnvironment: { REA_LOG_LEVEL: "silent" }, - createAnalysis: () => bindDirectAnalysis(dependencies), + createAnalysis: () => bindDirectAnalysis(runWith), registrations, }); @@ -344,14 +402,31 @@ mcpTest.for( OPERATIONS.map((operation) => [operation.contract.name, operation] as const), )( "%s returns the same Evidence through the CLI and MCP", - async ([name, { contract, description }], { mcp }) => { + async ([name, { contract, description }], { mcp, onTestFinished }) => { const directory = await createTestTempDirectory("rea-adapter-parity-"); - await writeFile(join(directory, "fixture"), minimalMachoImage(64)); + const realArtifact = + name === "inspect_artifact" || name === "extract_artifact"; + if (realArtifact) + await writeOrderedZip(join(directory, "fixture"), ["member.txt"]); + else await writeFile(join(directory, "fixture"), minimalMachoImage(64)); // Both adapters select the same file through the same noncanonical spelling. const target = `${directory}/./fixture`; - const input = callInput(contract, description, directory, target); + const input = { + ...callInput(contract, description, directory, target), + ...(name === "extract_artifact" + ? { output_root: join(directory, "extracted") } + : {}), + }; + const cli = createCli( + realArtifact + ? { + ...dependencies, + createArtifactBinarySession: () => createArtifactBinarySession({}), + } + : dependencies, + ); const cliResult = cliOutcome( - await serveInProcessCli(createCli(), argvFor(description, target, input)), + await serveInProcessCli(cli, argvFor(description, target, input)), target, ); // Each adapter writes caller-selected outputs into a fresh location. @@ -362,9 +437,13 @@ mcpTest.for( value.startsWith(`${directory}/`) ) await rm(value, { recursive: true, force: true }); + const mcpSession = realArtifact + ? await createArtifactBinarySession({}) + : session(); + if (realArtifact) ownArtifactSession(mcpSession, onTestFinished); const client = await mcp.connect( createServer( - { kind: "session", session: session() }, + { kind: "session", session: mcpSession }, { environment: { ...process.env, ...INLINE_EVIDENCE_MCP_ENVIRONMENT } }, ), ); @@ -377,13 +456,14 @@ mcpTest.for( await client.callTool({ name, arguments: input }), target, ); + if (realArtifact || authoredResults.has(name)) { + expect(cliResult.outcome, JSON.stringify(cliResult)).toBe("evidence"); + expect(mcpResult.outcome, JSON.stringify(mcpResult)).toBe("evidence"); + } const found = differences( cliResult as unknown as JsonValue, mcpResult as unknown as JsonValue, ); - // A differing outcome is one difference, not every field of both sides. - expect( - cliResult.outcome === mcpResult.outcome ? found : ["outcome"], - ).toEqual(KNOWN_DIFFERENCES[name] ?? []); + expect(found).toEqual(KNOWN_DIFFERENCES[name] ?? []); }, );