Files
reverse-engineering-anything/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
T
morluto 858ff73880 perf(tests): give MCP test clients the shared lazy schema validation (#10960)
* perf(tests): give MCP test clients the shared lazy schema validation

The MCP SDK client compiles every listed tool's output schema on its
first tool call. With the SDK's default Ajv options, REA's catalog costs
gigabytes of allocation per client (inspect_analysis_view alone: 1.7 s and
~500 MiB retained), so native MCP files peaked at 1-2 GiB per worker.

tests/fixtures/mcpSchemaValidation already owns lazy, reference-preserving
validation. Route the native MCP, process-boundary and shared harness
clients through it, share compiled validators across clients in a worker
(bounded LRU keyed by schema text), and add a counted architecture guard so
default-validation test clients cannot spread; the remaining copies are the
baseline.

* fix(guards): read call arguments past parentheses inside string literals

* perf(tests): give MCP boundary test clients the shared schema validation (#11022)

* perf(tests): give MCP boundary test clients the shared schema validation

* perf(tests): give the remaining MCP test clients the shared schema validation (#11025)

* fix(guards): read an MCP client's validator option from parsed code

The default-validation counter searched each new Client(...) call's text for jsonSchemaValidator, so an inline comment, a string literal or a client name naming the option hid a default client. Parse the source with @babel/parser and count NewExpressions of Client whose object-literal arguments have no jsonSchemaValidator property; a source that does not parse counts every textual call. The guard reads raw test sources, and the generated-child-script allowlist entry, which no longer counts as a call, is removed.

* fix(guards): read an MCP client's options only from a whole object-literal argument

The default-validation counter skipped files without the text
'new Client(', so a comment between the tokens hid a client, and it took
any depth-two key as an option, so 'enabled ? { jsonSchemaValidator } : {}'
passed while its false branch uses the SDK default. The skip now only
requires the name Client, and the validator counts only when it is a key
of an object literal that is a whole argument; anything else counts as the
default.
2026-11-15 21:53:53 -05:00

638 lines
21 KiB
TypeScript

import { unknownTarget } from "../../../src/domain/targetIdentity.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { expect, it, onTestFinished } from "vitest";
import { compareProcessCaptureFiles } from "../../support/cli/processComparisonCli.js";
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js";
import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCapture.fixture.js";
import {
capture,
partialSpecification,
processStarted,
terminal,
} from "../../../src/domain/process/processTraceComparison.fixture.js";
import {
parseProcessCapture,
type ProcessCapture,
} from "../../../src/domain/process/processCaptureParsing.js";
import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js";
import {
createEvidence,
parseEvidence,
type EvidenceProvider,
} from "../../../src/domain/evidence.js";
import { digestProcessCommitment } from "../../../src/domain/process/processScenario.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js";
const captureEvidence = (
side: "left" | "right",
comparisonContract: Readonly<Record<string, string>> = {},
provider: EvidenceProvider = PROCESS_PROVIDER,
observation: ProcessCapture = parseProcessCapture(
EMPTY_PROCESS_CAPTURE_EXAMPLE,
),
) =>
createEvidence(unknownTarget("not-observed"), provider, {
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: { side },
result: {
...observation,
manifest: {
...observation.manifest,
comparison_contract: comparisonContract,
comparison_contract_sha256: digestProcessCommitment(comparisonContract),
},
},
confidence: "observed",
authority: "controlled-replay",
});
const left = captureEvidence("left", { working_directory: "/work/reference" });
const right = captureEvidence("right", {
working_directory: "/work/reconstruction",
});
const connect = async (
session = createTestBinarySession(() => {
throw new Error("Process comparison must not launch a provider");
}),
maximumInputElements?: number,
) => {
const server = createServer(
{ kind: "session", session },
{
environment: {
...process.env,
...(maximumInputElements === undefined
? {}
: { REA_MCP_MAX_INPUT_ELEMENTS: String(maximumInputElements) }),
},
},
);
const client = new Client(
{ name: "process-comparison-test", version: "1" },
{ jsonSchemaValidator: createMcpSchemaValidation() },
);
onTestFinished(async () => {
await Promise.allSettled([client.close(), server.close(), session.close()]);
});
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
await server.connect(serverTransport);
await client.connect(clientTransport);
return client;
};
const compareViaCli = async (
left: ReturnType<typeof captureEvidence>,
right: ReturnType<typeof captureEvidence>,
specification?: ProcessTraceSpecification,
) => {
const root = await createTestTempDirectory("rea-process-provider-");
const leftPath = join(root, "left.json");
const rightPath = join(root, "right.json");
await Promise.all([
writeFile(leftPath, JSON.stringify(left)),
writeFile(rightPath, JSON.stringify(right)),
]);
const tracePath = join(root, "trace.json");
if (specification !== undefined)
await writeFile(tracePath, JSON.stringify(specification));
return compareProcessCaptureFiles(
leftPath,
rightPath,
specification === undefined ? undefined : tracePath,
);
};
// Each omission is checked once, alternating which side lacks the worker: a
// one-sided omission on either side, both sides, and neither.
it.each([
{ missingWorker: "left", omittedRendered: "left" },
{ missingWorker: "right", omittedRendered: "right" },
{ missingWorker: "right", omittedRendered: "both" },
{ missingWorker: "left", omittedRendered: "neither" },
] as const)(
"preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
async ({ missingWorker, omittedRendered }) => {
const observation = (side: "left" | "right") => {
const hasWorker = side !== missingWorker;
const complete = capture({
frames: [terminal],
process_samples: hasWorker ? [processStarted] : [],
event_journal: [
{ collection: "frames", index: 0 },
...(hasWorker
? [{ collection: "process_samples" as const, index: 0 }]
: []),
],
});
const omitted = omittedRendered === "both" || omittedRendered === side;
return captureEvidence(
side,
{},
PROCESS_PROVIDER,
omitted
? parseProcessCapture({
...complete,
truncated: true,
truncation_details: {
...complete.truncation_details,
rendered_terminal: {
budget_bytes: 5,
observed_bytes: 0,
retained_bytes: 0,
observed_frames: 1,
retained_frames: 0,
unknown_size_frames: 1,
},
},
})
: complete,
);
};
const left = observation("left");
const right = observation("right");
const terminalStatus =
omittedRendered === "neither" ? "unchanged" : "unknown";
const plain = parseEvidence(await compareViaCli(left, right));
expect(plain.normalized_result).toMatchObject({
terminal: terminalStatus,
process: missingWorker === "left" ? "added" : "removed",
status: "changed",
});
const specification = partialSpecification();
const cli = parseEvidence(await compareViaCli(left, right, specification));
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right, trace_spec: specification },
});
expect(response.isError).not.toBe(true);
expect(parseEvidence(response.structuredContent)).toEqual(cli);
expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
expect(cli.normalized_result).toMatchObject({
terminal: terminalStatus,
process: "changed",
status: "changed",
trace: {
verdict: "different",
diagnostic: {
kind: "cardinality",
side: missingWorker,
event_ids: ["worker"],
locations: [],
},
},
});
},
);
// Each provider field is checked once; alternating sides covers both labels.
it.each([
{ side: "left", change: { id: "unrelated-provider" } },
{ side: "right", change: { version: "1" } },
{ side: "left", change: { name: "unrelated provider" } },
] as const)(
"rejects an authenticated $side capture with provider $change",
async ({ side, change }) => {
const wrongProvider = captureEvidence(
side,
{},
{ ...PROCESS_PROVIDER, ...change },
);
expect(parseEvidence(wrongProvider)).toEqual(wrongProvider);
const left = side === "left" ? wrongProvider : captureEvidence("left");
const right = side === "right" ? wrongProvider : captureEvidence("right");
const issue = {
path: [side, "provider"],
reason: "invalid_value",
};
expect(await compareViaCli(left, right)).toMatchObject({
category: "invalid_input",
details: { operation: "compare_process_captures", issues: [issue] },
});
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right },
});
expect(response.isError).toBe(true);
expect(parseMcpToolError(response)).toMatchObject({
error: { code: "invalid_request", details: { issues: [issue] } },
});
await client.ping();
},
);
it.each([PROCESS_PROVIDER.name, "REA deterministic process harness"])(
"preserves authenticated CLI/MCP comparison Evidence for provider name %s",
async (name) => {
const provider = { ...PROCESS_PROVIDER, name };
const left = captureEvidence("left", {}, provider);
const right = captureEvidence("right", {}, provider);
const cli = parseEvidence(await compareViaCli(left, right));
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right },
});
expect(response.isError).not.toBe(true);
expect(parseEvidence(response.structuredContent)).toEqual(cli);
const text = response.content.find((block) => block.type === "text");
expect(text?.type).toBe("text");
if (text?.type !== "text") throw new Error("Missing comparison text");
expect(JSON.parse(text.text)).toEqual(cli);
expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
},
);
it.each(["left", "right"] as const)(
"authenticates the %s Evidence before provider admission",
async (side) => {
const mismatched = captureEvidence(
side,
{},
{ ...PROCESS_PROVIDER, id: "unrelated-provider" },
);
const tampered = { ...mismatched, parameters: { side: "tampered" } };
const left = side === "left" ? tampered : captureEvidence("left");
const right = side === "right" ? tampered : captureEvidence("right");
expect(await compareViaCli(left, right)).toMatchObject({
category: "invalid_input",
details: {
issues: [{ path: ["evidence_id"] }],
},
});
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right },
});
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
details: { side, issue_path: ["evidence_id"] },
},
});
},
);
it("names differing comparison contract fields through MCP and the CLI", async () => {
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right },
});
expect(response.isError).toBe(true);
const issue = {
path: ["right"],
reason: "invalid_value",
expected: ["working_directory"],
};
expect(parseMcpToolError(response)).toMatchObject({
error: { code: "invalid_request", details: { issues: [issue] } },
});
const root = await createTestTempDirectory("rea-process-contract-");
const leftPath = join(root, "left.json");
const rightPath = join(root, "right.json");
await writeFile(leftPath, JSON.stringify(left));
await writeFile(rightPath, JSON.stringify(right));
expect(await compareProcessCaptureFiles(leftPath, rightPath)).toMatchObject({
code: "invalid_request",
details: { operation: "compare_process_captures", issues: [issue] },
});
});
it("names the stale capture that exceeds max_capture_age_ms", async () => {
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: {
left: captureEvidence("left"),
right: captureEvidence("right"),
max_capture_age_ms: 1000,
},
});
expect(response.isError).toBe(true);
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "invalid_request",
details: {
issues: [
{
path: ["max_capture_age_ms"],
reason: "out_of_range",
message: expect.stringContaining(
`left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,
),
},
],
},
},
});
});
it("retains a selected max_capture_age_ms requirement in comparison Evidence parameters", async () => {
const session = createTestBinarySession(() => {
throw new Error("Process comparison must not launch a provider");
});
const client = await connect(session);
const unconstrained = await client.callTool({
name: "compare_process_captures",
arguments: {
left: captureEvidence("left"),
right: captureEvidence("right"),
},
});
const selected = await client.callTool({
name: "compare_process_captures",
arguments: {
left: captureEvidence("left"),
right: captureEvidence("right"),
max_capture_age_ms: Number.MAX_SAFE_INTEGER,
},
});
expect(unconstrained.isError).not.toBe(true);
expect(selected.isError).not.toBe(true);
const before = parseEvidence(unconstrained.structuredContent);
const after = parseEvidence(selected.structuredContent);
expect(before.parameters).not.toHaveProperty("max_capture_age_ms");
expect(after.parameters).toHaveProperty(
"max_capture_age_ms",
Number.MAX_SAFE_INTEGER,
);
expect(after.evidence_id).not.toBe(before.evidence_id);
expect(session.evidenceById(after.evidence_id)?.parameters).toHaveProperty(
"max_capture_age_ms",
Number.MAX_SAFE_INTEGER,
);
});
it("reports CLI trace specification issues under its traceSpec argument", async () => {
const root = await createTestTempDirectory("rea-process-trace-spec-");
const capturePath = join(root, "capture.json");
const specPath = join(root, "spec.json");
await writeFile(capturePath, JSON.stringify(captureEvidence("left")));
await writeFile(specPath, JSON.stringify({ language: {} }));
expect(
await compareProcessCaptureFiles(capturePath, capturePath, specPath),
).toMatchObject({
code: "invalid_request",
details: {
operation: "compare_process_captures",
issues: expect.arrayContaining([
expect.objectContaining({ path: ["traceSpec", "events"] }),
]),
},
});
});
it("rejects a deep trace cycle as an input error through CLI and MCP", async () => {
const events = Array.from({ length: 6000 }, (_, index) => ({
id: `e${String(index)}`,
source: "lifecycle",
exact: { marker: index },
}));
const trace = {
events,
language: {
kind: "partial_order",
happens_before: events.map((event, index) => ({
before: event.id,
after: events[(index + 1) % events.length]!.id,
})),
not_before: events.map((event, index) => ({
event: event.id,
anchor: events[(index + 1) % events.length]!.id,
})),
unordered_groups: [{ events: events.map(({ id }) => id) }],
},
};
// The CLI names the issue by its argument; MCP reports the same message.
const issue = {
path: ["traceSpec", "language"],
message: "happens-before graph must be acyclic",
};
const capture = captureEvidence("left");
const root = await createTestTempDirectory("rea-process-trace-cycle-");
const capturePath = join(root, "capture.json");
const specPath = join(root, "spec.json");
await writeFile(capturePath, JSON.stringify(capture));
await writeFile(specPath, JSON.stringify(trace));
expect(
await compareProcessCaptureFiles(capturePath, capturePath, specPath),
).toMatchObject({
code: "invalid_request",
details: { issues: [issue] },
});
// This 6,000-event graph intentionally exercises cycle validation beyond
// ordinary transport admission. Keep a finite explicit fixture allowance.
const client = await connect(undefined, 131_072);
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left: capture, right: capture, trace_spec: trace },
});
expect(response.isError).toBe(true);
expect(response.content).toEqual([
{ type: "text", text: expect.stringContaining(issue.message) },
]);
await client.ping();
});
it("names the side and constraint when Evidence is not a usable capture", async () => {
const client = await connect();
const capture = captureEvidence("right");
const unrelated = createEvidence(
unknownTarget("not-observed"),
PROCESS_PROVIDER,
{
operation: "other",
parameters: {},
result: {},
},
);
const wrongKind = await client.callTool({
name: "compare_process_captures",
arguments: { left: unrelated, right: capture },
});
expect(parseMcpToolError(wrongKind)).toMatchObject({
error: {
code: "invalid_request",
details: {
issues: [
{
path: ["left"],
reason: "invalid_value",
expected: {
operation: "capture_process_scenario",
predicate_type: "rea.process-capture",
},
observed: { operation: "other" },
},
],
},
},
});
const tampered = await client.callTool({
name: "compare_process_captures",
arguments: {
left: capture,
right: { ...capture, parameters: { side: "changed" } },
},
});
expect(parseMcpToolError(tampered)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
details: { side: "right", issue_path: ["evidence_id"] },
},
});
const coverage = EMPTY_PROCESS_CAPTURE_EXAMPLE.truncation_details;
const invalidCaptures = [
{
result: { manifest: {} },
issuePath: ["normalized_result", "manifest", "rea_version"],
},
{
result: {
...EMPTY_PROCESS_CAPTURE_EXAMPLE,
truncated: true,
truncation_details: {
...coverage,
rendered_terminal: {
...coverage.rendered_terminal,
observed_frames: 2,
unknown_size_frames: 1,
normalization_over_budget_frames: 2,
},
},
},
issuePath: [
"normalized_result",
"truncation_details",
"rendered_terminal",
"normalization_over_budget_frames",
],
},
];
const root = await createTestTempDirectory("rea-process-malformed-");
const leftPath = join(root, "left.json");
const rightPath = join(root, "right.json");
await writeFile(leftPath, JSON.stringify(capture));
for (const { result, issuePath } of invalidCaptures) {
const invalidResult = createEvidence(
unknownTarget("not-observed"),
PROCESS_PROVIDER,
{
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: {},
result: result,
},
);
expect(parseEvidence(invalidResult)).toEqual(invalidResult);
const malformed = await client.callTool({
name: "compare_process_captures",
arguments: { left: capture, right: invalidResult },
});
expect(parseMcpToolError(malformed)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
details: { side: "right", issue_path: issuePath },
},
});
await writeFile(rightPath, JSON.stringify(invalidResult));
expect(await compareProcessCaptureFiles(leftPath, rightPath)).toMatchObject(
{
code: "invalid_request",
category: "invalid_input",
details: {
issues: [
expect.objectContaining({
path: ["right"],
observed: expect.objectContaining({ issue_path: issuePath }),
}),
],
},
},
);
}
});
it("keeps structural capture issues alongside semantic ones through MCP and the CLI", async () => {
const client = await connect();
const capture = captureEvidence("left");
const {
manifest: { completed_at: _completedAt, ...manifest },
exit,
...rest
} = FINALIZED_PROCESS_CAPTURE_EXAMPLE;
const mixed = createEvidence(
unknownTarget("not-observed"),
PROCESS_PROVIDER,
{
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: {},
result: {
...rest,
manifest,
exit: {
...exit,
finalization: {
...exit.finalization,
signals: [
{ ...exit.finalization.signals[0], reason: "not allowed here" },
],
},
},
},
},
);
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left: capture, right: mixed },
});
expect(parseMcpToolError(response)).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
details: {
side: "right",
issue_path: ["normalized_result", "manifest", "completed_at"],
remaining_issue_count: 1,
},
},
});
expect(await compareViaCli(capture, mixed)).toMatchObject({
code: "invalid_request",
category: "invalid_input",
details: {
issues: [
{
path: ["right"],
observed: {
issue_path: ["normalized_result", "manifest", "completed_at"],
remaining_issue_count: 1,
},
},
],
},
});
});