diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index b179ac01cc..a9d10876a9 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -38,7 +38,15 @@ export const requireMcpOutput = ( throw new Error(`${operation} failed: ${mcpTextValue(result)}`); if (contract.name !== operation) throw new Error(`${operation} received the ${contract.name} contract`); - return contract.outputSchema.parse(jsonValue(result)); + const text = jsonValue(result); + const output = contract.outputSchema.parse(text); + if (result.structuredContent !== undefined) { + // The producer emits both representations from the same JSON value. + // Compare before schema projection can discard disagreeing source fields. + if (!isDeepStrictEqual(text, result.structuredContent)) + throw new Error(`${operation} text and structured results disagree`); + } + return output; }; const operationResultKey = (contract) => { @@ -124,15 +132,6 @@ export const requireMcpEvidenceBundle = (result) => { throw new Error( "get_evidence_bundle returned summary metadata instead of complete Evidence", ); - if (result.structuredContent !== undefined) { - const structured = contract.outputSchema.parse( - result.structuredContent, - ).result; - if (!isDeepStrictEqual(bundle, structured)) - throw new Error( - "get_evidence_bundle text and structured results disagree", - ); - } return authenticateParsedEvidenceBundle(bundle); }; diff --git a/scripts/verify-http3-network-captures.mjs b/scripts/verify-http3-network-captures.mjs index c601cac8bc..55997a95dc 100644 --- a/scripts/verify-http3-network-captures.mjs +++ b/scripts/verify-http3-network-captures.mjs @@ -26,7 +26,7 @@ import { } from "./lib/http3-verifier-lifecycle.mjs"; import { requireMcpEvidenceBundle, - mcpTextValue, + requireMcpOutput, } from "./lib/mcp-verifier-results.mjs"; const root = process.argv[2]; @@ -137,14 +137,11 @@ async function mcp(input, name) { join(directory, `${name}.mcp.reply.json`), JSON.stringify(reply, null, 2), ); - const value = JSON.parse(mcpTextValue(reply)); - assert.notEqual(reply.isError, true, mcpTextValue(reply)); + const evidence = requireMcpOutput(reply, contract.name, contract); assert.ok( validateOutput(reply.structuredContent), JSON.stringify(validateOutput.errors), ); - const evidence = contract.outputSchema.parse(value); - assert.deepEqual(reply.structuredContent, value); retainedEvidence.push(evidence); return evidence; } diff --git a/scripts/verify-packet-capture.mjs b/scripts/verify-packet-capture.mjs index 91ad2a8829..9e80b4ea8b 100644 --- a/scripts/verify-packet-capture.mjs +++ b/scripts/verify-packet-capture.mjs @@ -1,6 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpEvidenceBundle } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpEvidenceBundle, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -124,12 +128,13 @@ await withVerifierCleanup(async () => { name: tool.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(tool.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_packet_capture"), + ); const report = evidence.normalized_result; assert.equal( report.artifact.sha256, @@ -305,12 +310,13 @@ await withVerifierCleanup(async () => { name: pcapTool.name, arguments: { path: capturePath, detail: "summary" }, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( pcapAjv.validate(pcapTool.outputSchema, response.structuredContent), pcapAjv.errorsText(), ); - const summary = parseEvidence(response.structuredContent).normalized_result; + const summary = parseEvidence( + requireMcpOutput(response, "inspect_pcap_capture"), + ).normalized_result; assert.equal(summary.summary.capture_start, expectedCaptureStart); assert.equal(summary.summary.capture_end, expectedCaptureEnd); assert.equal( @@ -343,10 +349,9 @@ await withVerifierCleanup(async () => { view: { kind: "pcap", facet: "protocols", limit: 10 }, }, }); - assert.notEqual(page.isError, true, JSON.stringify(page)); assert.equal( - parseEvidence(page.structuredContent).normalized_result - .parent_evidence_id, + parseEvidence(requireMcpOutput(page, "inspect_analysis_view")) + .normalized_result.parent_evidence_id, source.evidence_id, ); cases++; @@ -368,19 +373,16 @@ await withVerifierCleanup(async () => { name: tool.name, arguments: input, }); - assert.equal(response.isError, true); + requireMcpToolError(response); await client.ping(); cases++; } await writeFile(path, "not a capture"); - assert.equal( - ( - await client.callTool({ - name: tool.name, - arguments: { path, fields: ["frame.number"] }, - }) - ).isError, - true, + requireMcpToolError( + await client.callTool({ + name: tool.name, + arguments: { path, fields: ["frame.number"] }, + }), ); cases++; const discovery = (await client.listTools()).tools.find( @@ -414,12 +416,13 @@ await withVerifierCleanup(async () => { name: discovery.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(discovery.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_packet_fields"), + ); const report = evidence.normalized_result; const kind = input.kind ?? "protocols"; const query = (input.query ?? "").toLowerCase(); diff --git a/scripts/verify-qlog-captures.mjs b/scripts/verify-qlog-captures.mjs index 8d1a16f6cd..3e82414b8d 100644 --- a/scripts/verify-qlog-captures.mjs +++ b/scripts/verify-qlog-captures.mjs @@ -10,6 +10,11 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; import { parseEvidence } from "../dist/domain/evidence.js"; +import { toolContract } from "../dist/contracts/toolContracts.js"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; // Caller-supplied producer evidence stays local; no fixture capture or secrets are bundled. const [controlPath, lossPath, reportPath] = process.argv.slice(2); @@ -40,6 +45,7 @@ transport.stderr?.on("data", (bytes) => { }); const ajv = new Ajv2020({ strict: false, validateFormats: false }); const measurements = []; +const contract = toolContract("inspect_web_network_capture"); try { try { await client.connect(transport); @@ -68,12 +74,13 @@ try { name: tool.name, arguments: input, }); - assert.notEqual(response.isError, true); + const mcp = parseEvidence( + requireMcpOutput(response, contract.name, contract), + ); assert.ok( validateOutput(response.structuredContent), JSON.stringify(validateOutput.errors), ); - const mcp = parseEvidence(response.structuredContent); const completed = await promisify(execFile)( process.execPath, [entrypoint, "inspect-web-network-capture", path, "qlog", "--json"], @@ -153,17 +160,14 @@ try { view: { kind: "api-surface" }, }, }); - assert.equal(retained.isError, true); - assert.equal( - JSON.parse(retained.content.find((item) => item.type === "text").text) - .error.code, - "unsupported_target", - ); + assert.equal(requireMcpToolError(retained).code, "unsupported_target"); const selected = await client.callTool({ name: tool.name, arguments: { ...input, record_ordinals: [capture.total_records - 1, 0] }, }); - const subset = parseEvidence(selected.structuredContent).normalized_result; + const subset = parseEvidence( + requireMcpOutput(selected, contract.name, contract), + ).normalized_result; assert.deepEqual( subset.records.map((record) => record.ordinal), [capture.total_records - 1, 0], diff --git a/scripts/verify-signal-recording.mjs b/scripts/verify-signal-recording.mjs index 1cafab175c..61d3afd831 100644 --- a/scripts/verify-signal-recording.mjs +++ b/scripts/verify-signal-recording.mjs @@ -1,6 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpEvidenceBundle } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpEvidenceBundle, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -47,9 +51,7 @@ const assertRejectedSelector = ( field, expectedExitCode = 1, ) => { - assert.equal(response.isError, true); - const text = response.content.find((item) => item.type === "text")?.text; - const { error } = JSON.parse(text ?? "{}"); + const error = requireMcpToolError(response); assert.equal(error.code, "invalid_request"); assert.equal(error.details.operation, operation); assert.deepEqual( @@ -123,12 +125,13 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(advertised.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "decode_signal_recording"), + ); const report = evidence.normalized_result; for (const byte of ["52", "45", "41"]) assert.match( @@ -238,12 +241,13 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(advertised.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "decode_signal_recording"), + ); const report = evidence.normalized_result; assert.deepEqual(report.output_window, { total_lines: 120, @@ -298,9 +302,9 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.equal(response.isError, true); if (rejectedField !== undefined) assertRejectedSelector(response, advertised.name, rejectedField); + else requireMcpToolError(response); await client.ping(); cases++; } @@ -309,7 +313,7 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: { path, decoders: "uart" }, }); - assert.equal(invalid.isError, true); + requireMcpToolError(invalid); cases++; const discoveryTool = (await client.listTools()).tools.find( ({ name }) => name === "inspect_signal_decoders", @@ -367,12 +371,13 @@ await withVerifierCleanup(async () => { name: discoveryTool.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(discoveryTool.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_signal_decoders"), + ); const report = evidence.normalized_result; discoveryMeasurements.push({ input, diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs index bd64b1ea80..3c8d813f30 100644 --- a/scripts/verify-web-network-captures.mjs +++ b/scripts/verify-web-network-captures.mjs @@ -18,7 +18,11 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { WEB_NETWORK_CAPTURE_LIMITS } from "../dist/domain/webNetworkCapture.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + mcpTextValue, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const executable = process.env.REA_MITMDUMP_COMMAND; @@ -903,15 +907,15 @@ async function inspect(mode, input, errorCategory) { name: "inspect_web_network_capture", arguments: input, }); - const value = JSON.parse(mcpTextValue(result)); if (errorCategory !== undefined) { - assert.equal(result.isError, true); - assert.equal(value.error.category, errorCategory); + const error = requireMcpToolError(result); + assert.equal(error.category, errorCategory); + const rawError = JSON.parse(mcpTextValue(result)).error; for (const literal of input.sensitive_values ?? []) - assert.ok(!JSON.stringify(value.error).includes(literal)); - return value.error; + assert.ok(!JSON.stringify(rawError).includes(literal)); + return error; } - assert.notEqual(result.isError, true, mcpTextValue(result)); + const value = requireMcpOutput(result, "inspect_web_network_capture"); assertSensitiveLimitations(value, input.sensitive_values ?? []); if (input.format === "har") { const surface = await client.callTool({ @@ -923,13 +927,12 @@ async function inspect(mode, input, errorCategory) { }); if ((input.sensitive_values ?? []).length > 0) { // Derived view text cannot be checked against undisclosed declarations. - assert.equal(surface.isError, true); + requireMcpToolError(surface); const refused = JSON.parse(mcpTextValue(surface)); for (const literal of input.sensitive_values) assert.ok(!JSON.stringify(refused).includes(literal)); } else { - assert.notEqual(surface.isError, true, mcpTextValue(surface)); - const projected = JSON.parse(mcpTextValue(surface)); + const projected = requireMcpOutput(surface, "inspect_analysis_view"); assert.equal(projected.normalized_result.kind, "api-surface"); assert.equal( projected.normalized_result.records.examined, @@ -944,13 +947,12 @@ async function inspect(mode, input, errorCategory) { }, }); if ((input.sensitive_values ?? []).length > 0) { - assert.equal(manifest.isError, true); + requireMcpToolError(manifest); const refusedManifest = JSON.parse(mcpTextValue(manifest)); for (const literal of input.sensitive_values) assert.ok(!JSON.stringify(refusedManifest).includes(literal)); } else { - assert.notEqual(manifest.isError, true, mcpTextValue(manifest)); - const projected = JSON.parse(mcpTextValue(manifest)); + const projected = requireMcpOutput(manifest, "inspect_analysis_view"); assert.equal(projected.normalized_result.kind, "environment-manifest"); assert.equal(projected.normalized_result.unknowns.length, 3); for (const record of projected.normalized_result.records) diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 9cda004246..1f0e4d5619 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -219,6 +219,62 @@ it("returns complete named Evidence with its subject, provider and analysis prov ); }); +it("admits matching complete text and structured Evidence and rejects either malformed shape", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect( + requireMcpOutput( + { ...textReply(evidence), structuredContent: evidence }, + "address_name", + ), + ).toEqual(evidence); + const malformed = { ...evidence, normalized_result: { name: "entry" } }; + for (const [text, structuredContent] of [ + [evidence, malformed], + [malformed, evidence], + ]) + expect(() => + requireMcpOutput( + { ...textReply(text), structuredContent }, + "address_name", + ), + ).toThrow(); +}); + +it("rejects disagreement in lifecycle envelope fields that the named schema would otherwise discard", async () => { + const { requireMcpOutput } = await verifierResults(); + const result = retainedBundle(); + const text = { result, vendor_note: "text observation" }; + const structuredContent = { result, vendor_note: "different observation" }; + const contract = toolContract("get_evidence_bundle"); + expect(contract.outputSchema.parse(text)).toEqual( + contract.outputSchema.parse(structuredContent), + ); + expect(() => + requireMcpOutput( + { ...textReply(text), structuredContent }, + "get_evidence_bundle", + ), + ).toThrow("text and structured results disagree"); +}); + +it("rejects disagreement in complete Evidence provenance before downstream extraction", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect(() => + requireMcpOutput( + { + ...textReply(evidence), + structuredContent: { + ...evidence, + raw_result: { engine_name: "different" }, + }, + }, + "address_name", + ), + ).toThrow("text and structured results disagree"); +}); + it("rejects mismatched named contracts, error-marked success and malformed normalized results", async () => { const { requireMcpOutput } = await verifierResults(); const evidence = namedEvidence();