mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 21:23:45 +00:00
test(parsers): compare adapters with admitted provider results (#12203)
This commit is contained in:
1 parent
feedbaede7
commit
63ea516832
1 file changed
+104
-24
@@ -13,6 +13,7 @@ import {
|
||||
import { createCliShell } from "../../../src/cli/cliShell.js";
|
||||
import type { ContractCliDescription } from "../../../src/cli/contractCliDescription.js";
|
||||
import { registeredContractDescriptions } from "../../support/cli/commandCli.js";
|
||||
import { createArtifactBinarySession } from "../../../src/composition/binary.js";
|
||||
import { bindDirectAnalysis } from "../../../src/composition/directAnalysis.js";
|
||||
import { parseConfig } from "../../../src/config/parseConfig.js";
|
||||
import { ARTIFACT_TOOL_CONTRACTS } from "../../../src/contracts/artifactToolContracts.js";
|
||||
@@ -25,6 +26,7 @@ import type { JsonValue } from "../../../src/domain/jsonValue.js";
|
||||
import { ok } from "../../../src/domain/result.js";
|
||||
import { createAnalysisExecution } from "../../../src/ports/AnalysisProvider.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js";
|
||||
import { createTestProviderRouter } from "../../fixtures/binarySession.js";
|
||||
import {
|
||||
minimalContractInput,
|
||||
@@ -33,20 +35,21 @@ import {
|
||||
import { minimalMachoImage } from "../../fixtures/nativeHeaders.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { serveInProcessCli } from "../../support/cli/serveInProcessCli.js";
|
||||
import { ownArtifactSession } from "../../support/mcp/artifactSession.js";
|
||||
import { mcpTest } from "../../support/mcp/mcpFixture.js";
|
||||
|
||||
/**
|
||||
* One call, two adapters: every analysis operation the CLI exposes runs once
|
||||
* through the CLI command tree and once through the MCP server, over the same
|
||||
* simulated provider and target. Both must deliver the same Evidence, or the
|
||||
* provider and target. Both must deliver the same Evidence, or the
|
||||
* same failure.
|
||||
*
|
||||
* KNOWN_DIFFERENCES records how main differs today. The W2 invoke path
|
||||
* removes them; when a change removes one, delete its entry. An entry never
|
||||
* grows. Both adapters must retain the exact caller-selected spelling.
|
||||
* KNOWN_DIFFERENCES records the exact Evidence and failure fields where the
|
||||
* adapters still differ. Both adapters must retain the caller’s exact spelling.
|
||||
*
|
||||
* Simulated providers return the smallest result each contract admits, so
|
||||
* this checks adapter composition, not real provider behavior.
|
||||
* Simulated providers use authored results where the JSON Schema cannot
|
||||
* express the complete runtime contract. Archive cases use the production
|
||||
* artifact reader over a small ZIP fixture.
|
||||
*/
|
||||
|
||||
/** JSON paths, under `evidence.` or `failure.`, where CLI and MCP differ. */
|
||||
@@ -64,6 +67,9 @@ const KNOWN_DIFFERENCES: Readonly<Record<string, readonly string[]>> = {
|
||||
"inspect_keyed_archive",
|
||||
"inspect_asset_catalog",
|
||||
"trace_dylib_resolution",
|
||||
"observe_native_ui",
|
||||
"capture_native_ui_scenario",
|
||||
"extract_artifact",
|
||||
].map((name) => [
|
||||
name,
|
||||
["evidence.evidence_id", "evidence.parameters.input_path"],
|
||||
@@ -83,16 +89,15 @@ const KNOWN_DIFFERENCES: Readonly<Record<string, readonly string[]>> = {
|
||||
],
|
||||
]),
|
||||
),
|
||||
// The MCP inspection records the delivery detail in Evidence parameters.
|
||||
inspect_artifact: [
|
||||
"evidence.evidence_id",
|
||||
"evidence.parameters.detail",
|
||||
"evidence.parameters.input_path",
|
||||
],
|
||||
// `DirectAnalysis` shapes CLI failures in the application layer and adds
|
||||
// an `error: "Analysis failed"` marker MCP does not send.
|
||||
trace_native_ui_action: ["failure.error"],
|
||||
// MCP checks the result against the contract output schema; the CLI
|
||||
// delivers Evidence whose result that schema refuses.
|
||||
read_bytes: ["outcome"],
|
||||
observe_native_ui: ["outcome"],
|
||||
capture_native_ui_scenario: ["outcome"],
|
||||
inspect_artifact: ["outcome"],
|
||||
extract_artifact: ["outcome"],
|
||||
};
|
||||
|
||||
/** Inputs a one-shot CLI call needs that MCP takes from session state. */
|
||||
@@ -105,8 +110,61 @@ const INPUT_OVERRIDES: Readonly<
|
||||
|
||||
const PROVIDER = { id: "fixture", name: "Fixture", version: "1" } as const;
|
||||
|
||||
/** Valid replies for contracts with relationships absent from JSON Schema. */
|
||||
const nativeUiResult = {
|
||||
target_sha256: "a".repeat(64),
|
||||
initial: {
|
||||
window: {
|
||||
pid: 123,
|
||||
window_id: 456,
|
||||
executable: "/fixture",
|
||||
launch_time: 0,
|
||||
title: "Fixture",
|
||||
},
|
||||
nodes: [],
|
||||
truncated: false,
|
||||
screenshot: null,
|
||||
gaps: [],
|
||||
},
|
||||
steps: [],
|
||||
restore: "leave-as-is",
|
||||
limitations: [],
|
||||
} satisfies JsonValue;
|
||||
|
||||
const authoredResults = new Map<string, JsonValue>([
|
||||
[
|
||||
"read_bytes",
|
||||
{
|
||||
address: "0x1000",
|
||||
requested_bytes: 16,
|
||||
returned_bytes: 1,
|
||||
bytes_hex: "ab",
|
||||
complete: false,
|
||||
},
|
||||
],
|
||||
["observe_native_ui", nativeUiResult],
|
||||
[
|
||||
"capture_native_ui_scenario",
|
||||
{
|
||||
...nativeUiResult,
|
||||
steps: [
|
||||
{
|
||||
index: 0,
|
||||
kind: "wait",
|
||||
before: nativeUiResult.initial,
|
||||
after: nativeUiResult.initial,
|
||||
outcome: "completed",
|
||||
reason: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
const results = new Map<string, JsonValue | undefined>();
|
||||
const resultFor = (operation: string): JsonValue | undefined => {
|
||||
const authored = authoredResults.get(operation);
|
||||
if (authored !== undefined) return authored;
|
||||
if (!results.has(operation)) {
|
||||
const contract = TOOL_CONTRACTS.find(({ name }) => name === operation);
|
||||
results.set(
|
||||
@@ -149,10 +207,10 @@ const registrations = await Promise.all(
|
||||
);
|
||||
|
||||
/** A fresh CLI command tree over the simulated provider. */
|
||||
const createCli = () =>
|
||||
const createCli = (runWith = dependencies) =>
|
||||
createCliShell({
|
||||
selectedEnvironment: { REA_LOG_LEVEL: "silent" },
|
||||
createAnalysis: () => bindDirectAnalysis(dependencies),
|
||||
createAnalysis: () => bindDirectAnalysis(runWith),
|
||||
registrations,
|
||||
});
|
||||
|
||||
@@ -344,14 +402,31 @@ mcpTest.for(
|
||||
OPERATIONS.map((operation) => [operation.contract.name, operation] as const),
|
||||
)(
|
||||
"%s returns the same Evidence through the CLI and MCP",
|
||||
async ([name, { contract, description }], { mcp }) => {
|
||||
async ([name, { contract, description }], { mcp, onTestFinished }) => {
|
||||
const directory = await createTestTempDirectory("rea-adapter-parity-");
|
||||
await writeFile(join(directory, "fixture"), minimalMachoImage(64));
|
||||
const realArtifact =
|
||||
name === "inspect_artifact" || name === "extract_artifact";
|
||||
if (realArtifact)
|
||||
await writeOrderedZip(join(directory, "fixture"), ["member.txt"]);
|
||||
else await writeFile(join(directory, "fixture"), minimalMachoImage(64));
|
||||
// Both adapters select the same file through the same noncanonical spelling.
|
||||
const target = `${directory}/./fixture`;
|
||||
const input = callInput(contract, description, directory, target);
|
||||
const input = {
|
||||
...callInput(contract, description, directory, target),
|
||||
...(name === "extract_artifact"
|
||||
? { output_root: join(directory, "extracted") }
|
||||
: {}),
|
||||
};
|
||||
const cli = createCli(
|
||||
realArtifact
|
||||
? {
|
||||
...dependencies,
|
||||
createArtifactBinarySession: () => createArtifactBinarySession({}),
|
||||
}
|
||||
: dependencies,
|
||||
);
|
||||
const cliResult = cliOutcome(
|
||||
await serveInProcessCli(createCli(), argvFor(description, target, input)),
|
||||
await serveInProcessCli(cli, argvFor(description, target, input)),
|
||||
target,
|
||||
);
|
||||
// Each adapter writes caller-selected outputs into a fresh location.
|
||||
@@ -362,9 +437,13 @@ mcpTest.for(
|
||||
value.startsWith(`${directory}/`)
|
||||
)
|
||||
await rm(value, { recursive: true, force: true });
|
||||
const mcpSession = realArtifact
|
||||
? await createArtifactBinarySession({})
|
||||
: session();
|
||||
if (realArtifact) ownArtifactSession(mcpSession, onTestFinished);
|
||||
const client = await mcp.connect(
|
||||
createServer(
|
||||
{ kind: "session", session: session() },
|
||||
{ kind: "session", session: mcpSession },
|
||||
{ environment: { ...process.env, ...INLINE_EVIDENCE_MCP_ENVIRONMENT } },
|
||||
),
|
||||
);
|
||||
@@ -377,13 +456,14 @@ mcpTest.for(
|
||||
await client.callTool({ name, arguments: input }),
|
||||
target,
|
||||
);
|
||||
if (realArtifact || authoredResults.has(name)) {
|
||||
expect(cliResult.outcome, JSON.stringify(cliResult)).toBe("evidence");
|
||||
expect(mcpResult.outcome, JSON.stringify(mcpResult)).toBe("evidence");
|
||||
}
|
||||
const found = differences(
|
||||
cliResult as unknown as JsonValue,
|
||||
mcpResult as unknown as JsonValue,
|
||||
);
|
||||
// A differing outcome is one difference, not every field of both sides.
|
||||
expect(
|
||||
cliResult.outcome === mcpResult.outcome ? found : ["outcome"],
|
||||
).toEqual(KNOWN_DIFFERENCES[name] ?? []);
|
||||
expect(found).toEqual(KNOWN_DIFFERENCES[name] ?? []);
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user