test(parsers): compare adapters with admitted provider results (#12203)

This commit is contained in:
morluto authored and GitHub committed 2026-11-19 05:20:35 +00:00
1 parent feedbaede7
commit 63ea516832
1 file changed
+104 -24
@@ -13,6 +13,7 @@ import {
import { createCliShell } from "../../../src/cli/cliShell.js";
import type { ContractCliDescription } from "../../../src/cli/contractCliDescription.js";
import { registeredContractDescriptions } from "../../support/cli/commandCli.js";
import { createArtifactBinarySession } from "../../../src/composition/binary.js";
import { bindDirectAnalysis } from "../../../src/composition/directAnalysis.js";
import { parseConfig } from "../../../src/config/parseConfig.js";
import { ARTIFACT_TOOL_CONTRACTS } from "../../../src/contracts/artifactToolContracts.js";
@@ -25,6 +26,7 @@ import type { JsonValue } from "../../../src/domain/jsonValue.js";
import { ok } from "../../../src/domain/result.js";
import { createAnalysisExecution } from "../../../src/ports/AnalysisProvider.js";
import { createServer } from "../../../src/server/createServer.js";
import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js";
import { createTestProviderRouter } from "../../fixtures/binarySession.js";
import {
minimalContractInput,
@@ -33,20 +35,21 @@ import {
import { minimalMachoImage } from "../../fixtures/nativeHeaders.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { serveInProcessCli } from "../../support/cli/serveInProcessCli.js";
import { ownArtifactSession } from "../../support/mcp/artifactSession.js";
import { mcpTest } from "../../support/mcp/mcpFixture.js";
/**
* One call, two adapters: every analysis operation the CLI exposes runs once
* through the CLI command tree and once through the MCP server, over the same
* simulated provider and target. Both must deliver the same Evidence, or the
* provider and target. Both must deliver the same Evidence, or the
* same failure.
*
* KNOWN_DIFFERENCES records how main differs today. The W2 invoke path
* removes them; when a change removes one, delete its entry. An entry never
* grows. Both adapters must retain the exact caller-selected spelling.
* KNOWN_DIFFERENCES records the exact Evidence and failure fields where the
* adapters still differ. Both adapters must retain the caller’s exact spelling.
*
* Simulated providers return the smallest result each contract admits, so
* this checks adapter composition, not real provider behavior.
* Simulated providers use authored results where the JSON Schema cannot
* express the complete runtime contract. Archive cases use the production
* artifact reader over a small ZIP fixture.
*/
/** JSON paths, under `evidence.` or `failure.`, where CLI and MCP differ. */
@@ -64,6 +67,9 @@ const KNOWN_DIFFERENCES: Readonly<Record<string, readonly string[]>> = {
"inspect_keyed_archive",
"inspect_asset_catalog",
"trace_dylib_resolution",
"observe_native_ui",
"capture_native_ui_scenario",
"extract_artifact",
].map((name) => [
name,
["evidence.evidence_id", "evidence.parameters.input_path"],
@@ -83,16 +89,15 @@ const KNOWN_DIFFERENCES: Readonly<Record<string, readonly string[]>> = {
],
]),
),
// The MCP inspection records the delivery detail in Evidence parameters.
inspect_artifact: [
"evidence.evidence_id",
"evidence.parameters.detail",
"evidence.parameters.input_path",
],
// `DirectAnalysis` shapes CLI failures in the application layer and adds
// an `error: "Analysis failed"` marker MCP does not send.
trace_native_ui_action: ["failure.error"],
// MCP checks the result against the contract output schema; the CLI
// delivers Evidence whose result that schema refuses.
read_bytes: ["outcome"],
observe_native_ui: ["outcome"],
capture_native_ui_scenario: ["outcome"],
inspect_artifact: ["outcome"],
extract_artifact: ["outcome"],
};
/** Inputs a one-shot CLI call needs that MCP takes from session state. */
@@ -105,8 +110,61 @@ const INPUT_OVERRIDES: Readonly<
const PROVIDER = { id: "fixture", name: "Fixture", version: "1" } as const;
/** Valid replies for contracts with relationships absent from JSON Schema. */
const nativeUiResult = {
target_sha256: "a".repeat(64),
initial: {
window: {
pid: 123,
window_id: 456,
executable: "/fixture",
launch_time: 0,
title: "Fixture",
},
nodes: [],
truncated: false,
screenshot: null,
gaps: [],
},
steps: [],
restore: "leave-as-is",
limitations: [],
} satisfies JsonValue;
const authoredResults = new Map<string, JsonValue>([
[
"read_bytes",
{
address: "0x1000",
requested_bytes: 16,
returned_bytes: 1,
bytes_hex: "ab",
complete: false,
},
],
["observe_native_ui", nativeUiResult],
[
"capture_native_ui_scenario",
{
...nativeUiResult,
steps: [
{
index: 0,
kind: "wait",
before: nativeUiResult.initial,
after: nativeUiResult.initial,
outcome: "completed",
reason: null,
},
],
},
],
]);
const results = new Map<string, JsonValue | undefined>();
const resultFor = (operation: string): JsonValue | undefined => {
const authored = authoredResults.get(operation);
if (authored !== undefined) return authored;
if (!results.has(operation)) {
const contract = TOOL_CONTRACTS.find(({ name }) => name === operation);
results.set(
@@ -149,10 +207,10 @@ const registrations = await Promise.all(
);
/** A fresh CLI command tree over the simulated provider. */
const createCli = () =>
const createCli = (runWith = dependencies) =>
createCliShell({
selectedEnvironment: { REA_LOG_LEVEL: "silent" },
createAnalysis: () => bindDirectAnalysis(dependencies),
createAnalysis: () => bindDirectAnalysis(runWith),
registrations,
});
@@ -344,14 +402,31 @@ mcpTest.for(
OPERATIONS.map((operation) => [operation.contract.name, operation] as const),
)(
"%s returns the same Evidence through the CLI and MCP",
async ([name, { contract, description }], { mcp }) => {
async ([name, { contract, description }], { mcp, onTestFinished }) => {
const directory = await createTestTempDirectory("rea-adapter-parity-");
await writeFile(join(directory, "fixture"), minimalMachoImage(64));
const realArtifact =
name === "inspect_artifact" || name === "extract_artifact";
if (realArtifact)
await writeOrderedZip(join(directory, "fixture"), ["member.txt"]);
else await writeFile(join(directory, "fixture"), minimalMachoImage(64));
// Both adapters select the same file through the same noncanonical spelling.
const target = `${directory}/./fixture`;
const input = callInput(contract, description, directory, target);
const input = {
...callInput(contract, description, directory, target),
...(name === "extract_artifact"
? { output_root: join(directory, "extracted") }
: {}),
};
const cli = createCli(
realArtifact
? {
...dependencies,
createArtifactBinarySession: () => createArtifactBinarySession({}),
}
: dependencies,
);
const cliResult = cliOutcome(
await serveInProcessCli(createCli(), argvFor(description, target, input)),
await serveInProcessCli(cli, argvFor(description, target, input)),
target,
);
// Each adapter writes caller-selected outputs into a fresh location.
@@ -362,9 +437,13 @@ mcpTest.for(
value.startsWith(`${directory}/`)
)
await rm(value, { recursive: true, force: true });
const mcpSession = realArtifact
? await createArtifactBinarySession({})
: session();
if (realArtifact) ownArtifactSession(mcpSession, onTestFinished);
const client = await mcp.connect(
createServer(
{ kind: "session", session: session() },
{ kind: "session", session: mcpSession },
{ environment: { ...process.env, ...INLINE_EVIDENCE_MCP_ENVIRONMENT } },
),
);
@@ -377,13 +456,14 @@ mcpTest.for(
await client.callTool({ name, arguments: input }),
target,
);
if (realArtifact || authoredResults.has(name)) {
expect(cliResult.outcome, JSON.stringify(cliResult)).toBe("evidence");
expect(mcpResult.outcome, JSON.stringify(mcpResult)).toBe("evidence");
}
const found = differences(
cliResult as unknown as JsonValue,
mcpResult as unknown as JsonValue,
);
// A differing outcome is one difference, not every field of both sides.
expect(
cliResult.outcome === mcpResult.outcome ? found : ["outcome"],
).toEqual(KNOWN_DIFFERENCES[name] ?? []);
expect(found).toEqual(KNOWN_DIFFERENCES[name] ?? []);
},
);