diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index 3c98dbc8cb..b179ac01cc 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -28,7 +28,12 @@ export const requireMcpToolError = (result) => { return analysisErrorProjectionSchema.parse(jsonValue(result).error); }; -const requireOutput = (result, operation, contract) => { +/** Validate the complete output against its exact named contract, retaining inline Evidence. */ +export const requireMcpOutput = ( + result, + operation, + contract = toolContract(operation), +) => { if (result.isError === true) throw new Error(`${operation} failed: ${mcpTextValue(result)}`); if (contract.name !== operation) @@ -96,7 +101,7 @@ export const requireMcpEvidenceResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "normalized_result") throw new Error(`${operation} requires the lifecycle result extractor`); - return requireOutput(result, operation, contract).normalized_result; + return requireMcpOutput(result, operation, contract).normalized_result; }; /** Validate a lifecycle tool output and return its operation result. */ @@ -104,7 +109,7 @@ export const requireMcpLifecycleResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "result") throw new Error(`${operation} requires the Evidence result extractor`); - const projected = requireOutput(result, operation, contract).result; + const projected = requireMcpOutput(result, operation, contract).result; return operation === "binary_session" ? expandMcpSessionStatus(projected) : projected; @@ -114,7 +119,7 @@ export const requireMcpLifecycleResult = (result, operation) => { export const requireMcpEvidenceBundle = (result) => { const operation = "get_evidence_bundle"; const contract = toolContract(operation); - const bundle = requireOutput(result, operation, contract).result; + const bundle = requireMcpOutput(result, operation, contract).result; if (bundle.kind === "evidence-bundle-summary") throw new Error( "get_evidence_bundle returned summary metadata instead of complete Evidence", @@ -138,7 +143,7 @@ export const requireMcpOperationResult = ( contract = toolContract(operation), ) => { const key = operationResultKey(contract); - const projected = requireOutput(result, operation, contract)[key]; + const projected = requireMcpOutput(result, operation, contract)[key]; return operation === "binary_session" ? expandMcpSessionStatus(projected) : projected; diff --git a/scripts/verify-binary-layout.mjs b/scripts/verify-binary-layout.mjs index dd004a06d2..64a1b430f4 100644 --- a/scripts/verify-binary-layout.mjs +++ b/scripts/verify-binary-layout.mjs @@ -20,7 +20,10 @@ import { runOwnedCommand, } from "../dist/process/OwnedCommand.js"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { sectionNameFixtures } from "./lib/elf-section-name-fixtures.mjs"; import { withLargeResultMcp } from "./lib/large-result-mcp.mjs"; @@ -1549,14 +1552,12 @@ async function inspect( }, { timeout: 300_000 }, ); - const value = JSON.parse(mcpTextValue(response)); if (category !== undefined) { - assert.equal(response.isError, true); - assert.equal(value.error.category, category, JSON.stringify(value.error)); - return value.error; + const error = requireMcpToolError(response); + assert.equal(error.category, category, JSON.stringify(error)); + return error; } - assert.notEqual(response.isError, true, mcpTextValue(response)); - envelope = value; + envelope = requireMcpOutput(response, "inspect_binary_layout"); } else { let response; try { diff --git a/scripts/verify-evm-interface.mjs b/scripts/verify-evm-interface.mjs index 4bacc4bdfd..cec0526227 100644 --- a/scripts/verify-evm-interface.mjs +++ b/scripts/verify-evm-interface.mjs @@ -11,7 +11,10 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const solcModule = process.env.REA_VERIFY_SOLC_MODULE; @@ -103,11 +106,10 @@ try { name: "inspect_evm_interface", arguments: { path, encoding }, }); - value = JSON.parse(mcpTextValue(reply)); - if (expectedCategory !== undefined) { - assert.equal(reply.isError, true); - value = value.error; - } else assert.notEqual(reply.isError, true, mcpTextValue(reply)); + value = + expectedCategory !== undefined + ? requireMcpToolError(reply) + : requireMcpOutput(reply, "inspect_evm_interface"); } if (expectedCategory !== undefined) { assert.equal(value.category, expectedCategory, JSON.stringify(value)); diff --git a/scripts/verify-flutter.mjs b/scripts/verify-flutter.mjs index 84d49e0862..b20d76d72b 100644 --- a/scripts/verify-flutter.mjs +++ b/scripts/verify-flutter.mjs @@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { toolContract } from "../dist/contracts/toolContracts.js"; import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs"; import { closeVerifierResources, withVerifierCleanup, @@ -234,8 +234,7 @@ async function run(operation, input) { { name: operation, arguments: input }, { timeout: 60_000 }, ); - assert.notEqual(response.isError, true, mcpTextValue(response)); - const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response))); + const mcp = requireMcpOutput(response, operation, contract); assert.deepEqual(cli.normalized_result, mcp.normalized_result); assert.deepEqual(cli.subject, mcp.subject); assert.equal(cli.subject.local_path, input.path); diff --git a/scripts/verify-go-binary.mjs b/scripts/verify-go-binary.mjs index 4af896656f..799b21cecf 100644 --- a/scripts/verify-go-binary.mjs +++ b/scripts/verify-go-binary.mjs @@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; import { goBinarySchema } from "../dist/domain/go/goBinary.js"; import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const execute = promisify(execFile); @@ -359,8 +359,7 @@ async function inspect(mode, path) { { name: "inspect_go_binary", arguments: { path } }, { timeout: 30_000 }, ); - assert.notEqual(response.isError, true, mcpTextValue(response)); - envelope = JSON.parse(mcpTextValue(response)); + envelope = requireMcpOutput(response, "inspect_go_binary"); } const evidence = parseEvidence(envelope); assert.equal(evidence.subject.local_path, path); diff --git a/scripts/verify-il2cpp-metadata.mjs b/scripts/verify-il2cpp-metadata.mjs index dd002a3ec5..5bc7d1d6f8 100644 --- a/scripts/verify-il2cpp-metadata.mjs +++ b/scripts/verify-il2cpp-metadata.mjs @@ -6,9 +6,11 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { absoluteCallerPath } from "../dist/filesystem/callerPath.js"; import { toolContract } from "../dist/contracts/toolContracts.js"; -import { analysisViewResultSchema } from "../dist/domain/analysisView/analysisView.js"; import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { hashVerifierFile } from "./lib/verifier-file-resources.mjs"; @@ -43,8 +45,7 @@ const call = async (name, input) => { { name, arguments: input }, { timeout: 120_000 }, ); - assert.notEqual(response.isError, true, mcpTextValue(response)); - return JSON.parse(mcpTextValue(response)); + return requireMcpOutput(response, name); }; const resources = new ArtifactResourceScope(); await withVerifierCleanup(async () => { @@ -74,7 +75,7 @@ await withVerifierCleanup(async () => { if (complete.isError === true) { // Large inventories exceed the stdio budget; the result must stay // retained and addressable rather than truncated. - const { error } = JSON.parse(mcpTextValue(complete)); + const error = requireMcpToolError(complete); assert.equal(error.code, "resource_constraint"); assert.equal(error.details.reported_limits.boundary, "mcp-response"); assert.match(error.details.reported_limits.evidence_id, /^ev_/u); @@ -82,9 +83,7 @@ await withVerifierCleanup(async () => { `[ok] MCP complete delivery refused at the stdio budget (${error.details.reported_limits.response_bytes_at_least} bytes) with retained Evidence`, ); } else { - const mcp = contract.outputSchema.parse( - JSON.parse(mcpTextValue(complete)), - ); + const mcp = requireMcpOutput(complete, operation, contract); assert.deepEqual(mcp.normalized_result, report); console.log("[ok] CLI/MCP complete Evidence parity"); } @@ -108,9 +107,7 @@ await withVerifierCleanup(async () => { }); console.log("[ok] image type ranges and type method ranges agree"); - const summary = contract.outputSchema.parse( - await call(operation, { path }), - ).normalized_result; + const summary = (await call(operation, { path })).normalized_result; assert.equal(summary.summary.magic, report.magic); assert.deepEqual(summary.summary.counts, { sections: report.sections.length, @@ -135,37 +132,33 @@ await withVerifierCleanup(async () => { generic_method_instance_states: null, generic_method_issues: null, }); - const page = analysisViewResultSchema.parse( - ( - await call("inspect_analysis_view", { - source: { - kind: "retained-evidence", - evidence_id: summary.parent_evidence_id, - }, - view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 }, - }) - ).normalized_result, - ); + const page = ( + await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 }, + }) + ).normalized_result; assert.deepEqual( page.items, report.methods.slice(0, 16).map((row, index) => ({ ...row, index })), ); - const last = analysisViewResultSchema.parse( - ( - await call("inspect_analysis_view", { - source: { - kind: "retained-evidence", - evidence_id: summary.parent_evidence_id, - }, - view: { - kind: "il2cpp-metadata", - collection: "types", - offset: Math.max(0, report.types.length - 4), - limit: 16, - }, - }) - ).normalized_result, - ); + const last = ( + await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { + kind: "il2cpp-metadata", + collection: "types", + offset: Math.max(0, report.types.length - 4), + limit: 16, + }, + }) + ).normalized_result; const lastOffset = Math.max(0, report.types.length - 4); assert.deepEqual( last.items, diff --git a/scripts/verify-pcap.mjs b/scripts/verify-pcap.mjs index c5376a7c20..6c0bdf5eb0 100644 --- a/scripts/verify-pcap.mjs +++ b/scripts/verify-pcap.mjs @@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { toolContract } from "../dist/contracts/toolContracts.js"; import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs"; import { closeVerifierResources, withVerifierCleanup, @@ -123,8 +123,7 @@ async function run(name, input) { { name, arguments: input }, { timeout: 360_000 }, ); - assert.notEqual(response.isError, true, mcpTextValue(response)); - const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response))); + const mcp = requireMcpOutput(response, name, contract); assert.deepEqual(mcp.normalized_result, cli.normalized_result); assert.deepEqual(mcp.subject, cli.subject); if (input.path !== undefined) { diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 5ff69e866b..9cda004246 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -1,3 +1,5 @@ +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { createEvidenceBundle, @@ -6,6 +8,12 @@ import { import { expect, it } from "vitest"; interface VerifierResults { + readonly requireMcpOutput: ( + result: unknown, + operation: string, + contract?: ToolContract, + ) => unknown; + readonly requireMcpToolError: (result: unknown) => unknown; readonly requireMcpEvidenceBundle: (result: unknown) => EvidenceBundle; readonly requireMcpLifecycleResult: ( result: unknown, @@ -182,3 +190,79 @@ it("retains summary discovery in the generic lifecycle reader but refuses it as "summary metadata instead of complete Evidence", ); }); + +const namedEvidence = () => + createEvidence( + { + path: "/fixture/image.bin", + sha256: "a".repeat(64), + format: "elf", + architecture: "x86_64", + }, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "address_name", + parameters: { address: "0x1000" }, + result: "entry", + rawResult: { engine_name: "entry", source: "symbol table" }, + }, + ); +const textReply = (value: unknown) => ({ + content: [{ type: "text", text: JSON.stringify(value) }], +}); + +it("returns complete named Evidence with its subject, provider and analysis provenance", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect(requireMcpOutput(textReply(evidence), "address_name")).toEqual( + evidence, + ); +}); + +it("rejects mismatched named contracts, error-marked success and malformed normalized results", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect(() => + requireMcpOutput( + textReply(evidence), + "address_name", + toolContract("set_address_name"), + ), + ).toThrow("received the set_address_name contract"); + expect(() => + requireMcpOutput({ ...textReply(evidence), isError: true }, "address_name"), + ).toThrow("address_name failed"); + expect(() => + requireMcpOutput( + textReply({ ...evidence, normalized_result: { name: "entry" } }), + "address_name", + ), + ).toThrow(); +}); + +it("requires canonical complete tool errors before inspecting their diagnostic fields", async () => { + const { requireMcpToolError } = await verifierResults(); + const error = { + code: "invalid_request", + category: "invalid_input", + message: "The address is invalid", + retryable: false, + remediation: { action: "Choose an observed address" }, + details: { operation: "address_name", address: "bad" }, + }; + expect( + requireMcpToolError({ ...textReply({ error }), isError: true }), + ).toEqual(error); + for (const value of [ + { error: { category: "invalid_input" } }, + error, + { error: { ...error, retryable: "false" } }, + { error: { ...error, code: "vendor_failure" } }, + ]) + expect(() => + requireMcpToolError({ ...textReply(value), isError: true }), + ).toThrow(); + expect(() => requireMcpToolError(textReply({ error }))).toThrow( + "Expected an MCP tool failure", + ); +});