diff --git a/scripts/verify-real-ghidra-6502.mjs b/scripts/verify-real-ghidra-6502.mjs index 87fb01d410..3417a9c7ff 100644 --- a/scripts/verify-real-ghidra-6502.mjs +++ b/scripts/verify-real-ghidra-6502.mjs @@ -18,7 +18,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -464,10 +467,9 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); if (["open_binary", "binary_session", "close_binary"].includes(name)) return requireMcpOperationResult(result, name); - return result.structuredContent; + return requireMcpOutput(result, name); } function evidence(value, sha256) { diff --git a/scripts/verify-real-ghidra-address-context.mjs b/scripts/verify-real-ghidra-address-context.mjs index 432680b299..3d9cac7d4b 100644 --- a/scripts/verify-real-ghidra-address-context.mjs +++ b/scripts/verify-real-ghidra-address-context.mjs @@ -9,7 +9,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2 || process.platform !== "linux") @@ -320,8 +323,7 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); return ["open_binary", "binary_session", "close_binary"].includes(name) ? requireMcpOperationResult(result, name) - : result.structuredContent; + : requireMcpOutput(result, name); } diff --git a/scripts/verify-real-ghidra-com.mjs b/scripts/verify-real-ghidra-com.mjs index c63b3b0222..6139f633c2 100644 --- a/scripts/verify-real-ghidra-com.mjs +++ b/scripts/verify-real-ghidra-com.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; @@ -339,16 +342,11 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(value.isError, true, JSON.stringify(value)); - const projected = requireMcpOperationResult(value, name); - if (name === "open_binary" || name === "close_binary") return projected; - const evidence = parseEvidence(value.structuredContent); + if (["open_binary", "close_binary"].includes(name)) + return requireMcpLifecycleResult(value, name); + const evidence = parseEvidence(requireMcpOutput(value, name)); assert.equal(evidence.subject.digest.sha256, sha256); - assert.deepEqual( - evidence.normalized_result, - value.structuredContent.normalized_result, - ); - return projected; + return evidence.normalized_result; } async function cli(command, address, options = []) { const { stdout } = await promisify(execFile)( diff --git a/scripts/verify-real-ghidra-dos.mjs b/scripts/verify-real-ghidra-dos.mjs index ec49ca8499..1c685f8d97 100644 --- a/scripts/verify-real-ghidra-dos.mjs +++ b/scripts/verify-real-ghidra-dos.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -555,7 +558,6 @@ async function call(name, arguments_ = {}) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); assert.ok(result.structuredContent, `${name} omitted structured result`); return requireMcpOperationResult(result, name); } @@ -564,8 +566,7 @@ async function evidenceCall(name, arguments_) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); - const evidence = parseEvidence(result.structuredContent); + const evidence = parseEvidence(requireMcpOutput(result, name)); assert.equal(evidence.subject.digest.sha256, sha256); assert.equal(evidence.provider.id, "ghidra"); assertProfile(evidence.analysis_profile); diff --git a/scripts/verify-real-ghidra-m68k.mjs b/scripts/verify-real-ghidra-m68k.mjs index 537ebdf6cc..17d9b26b65 100644 --- a/scripts/verify-real-ghidra-m68k.mjs +++ b/scripts/verify-real-ghidra-m68k.mjs @@ -16,7 +16,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -281,10 +284,9 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); if (["open_binary", "binary_session", "close_binary"].includes(name)) return requireMcpOperationResult(result, name); - return result.structuredContent; + return requireMcpOutput(result, name); } function evidence(value) { diff --git a/scripts/verify-real-ghidra-nativeaot.mjs b/scripts/verify-real-ghidra-nativeaot.mjs index e6280f601a..fb5064613f 100644 --- a/scripts/verify-real-ghidra-nativeaot.mjs +++ b/scripts/verify-real-ghidra-nativeaot.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -394,14 +397,8 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 360000 }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - const projected = requireMcpOperationResult(response, name); - if (name === "close_binary") return projected; - const evidence = parseEvidence(response.structuredContent); + if (name === "close_binary") return requireMcpLifecycleResult(response, name); + const evidence = parseEvidence(requireMcpOutput(response, name)); assert.equal(evidence.subject.digest.sha256, sha256); - assert.deepEqual( - evidence.normalized_result, - response.structuredContent.normalized_result, - ); - return projected; + return evidence.normalized_result; } diff --git a/scripts/verify-real-ghidra-raw.mjs b/scripts/verify-real-ghidra-raw.mjs index 549da44456..1b09cb60c1 100644 --- a/scripts/verify-real-ghidra-raw.mjs +++ b/scripts/verify-real-ghidra-raw.mjs @@ -12,7 +12,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-inspection.mjs"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -351,15 +354,9 @@ async function call(name, args = {}, targetSha256 = sha256) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); - const projected = requireMcpOperationResult(result, name); - if ( - name === "open_binary" || - name === "close_binary" || - name === "binary_session" - ) - return projected; - const evidence = parseEvidence(result.structuredContent); + if (["open_binary", "close_binary", "binary_session"].includes(name)) + return requireMcpLifecycleResult(result, name); + const evidence = parseEvidence(requireMcpOutput(result, name)); assert.equal(evidence.subject.digest.sha256, targetSha256); assert.equal(evidence.subject.format, "raw-binary"); return evidence; diff --git a/scripts/verify-real-ghidra-riscv.mjs b/scripts/verify-real-ghidra-riscv.mjs index 766b064434..29e67d88fa 100644 --- a/scripts/verify-real-ghidra-riscv.mjs +++ b/scripts/verify-real-ghidra-riscv.mjs @@ -26,7 +26,8 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins import { buildRiscvFixture } from "../tests/conformance/ghidra/riscv-fixture.mjs"; import { mcpTextValue, - requireMcpOperationResult, + requireMcpLifecycleResult, + requireMcpOutput, requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -141,18 +142,24 @@ try { { name, arguments: args }, { timeout: 240000 }, ); - const projected = requireMcpOperationResult(reply, name); + const lifecycle = [ + "open_binary", + "close_binary", + "binary_session", + ].includes(name); + const admitted = lifecycle + ? requireMcpLifecycleResult(reply, name) + : parseEvidence(requireMcpOutput(reply, name)); const textual = JSON.parse(mcpTextValue(reply)); - assert.deepEqual(reply.structuredContent, textual); + assert.ok(reply.structuredContent, `${name} omitted structured result`); const validate = validators.get(name); assert.ok(validate, `${name} missing from advertised catalog`); assert.ok( validate(textual), `${name}: ${JSON.stringify(validate.errors)}`, ); - if (!["open_binary", "close_binary", "binary_session"].includes(name)) - assertEvidence(parseEvidence(textual)); - return projected; + if (!lifecycle) assertEvidence(admitted); + return lifecycle ? admitted : admitted.normalized_result; }; const opened = await call("open_binary", { path: fixture.path, diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 09d92c4fb1..7b2c02e04f 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -21,7 +21,6 @@ import { ProviderProcessSupervisor, spawnOwnedProviderProcess, } from "../dist/process/ProviderProcess.js"; -import { toolContract } from "../dist/contracts/toolContracts.js"; import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -215,7 +214,7 @@ async function call(name, arguments_) { { timeout: 180_000 }, ); if (name === "analyze_swift_types") - toolContract(name).outputSchema.parse(response.structuredContent); + assert.ok(response.structuredContent, `${name} omitted structured result`); return requireMcpOperationResult(response, name); } diff --git a/scripts/verify-real-ghidra-switch.mjs b/scripts/verify-real-ghidra-switch.mjs index 5d43c2bc4c..5797969234 100644 --- a/scripts/verify-real-ghidra-switch.mjs +++ b/scripts/verify-real-ghidra-switch.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -326,9 +329,8 @@ async function toolResult(client, name, arguments_ = {}) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); assert.ok(result.structuredContent, `${name} omitted structured result`); - return result.structuredContent; + return requireMcpOutput(result, name); } async function call(client, name, arguments_ = {}) { const result = await client.callTool( diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 1f0e4d5619..69c9ced8b5 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -1,6 +1,12 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { runInNewContext } from "node:vm"; +import { parse } from "@babel/parser"; import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; -import { createEvidence } from "../../../src/domain/evidence.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { SUPPORTED_GHIDRA_VERSION } from "../../../src/ghidra/GhidraReleasePolicy.js"; import { createEvidenceBundle, type EvidenceBundle, @@ -8,6 +14,10 @@ import { import { expect, it } from "vitest"; interface VerifierResults { + readonly requireMcpOperationResult: ( + result: unknown, + operation: string, + ) => unknown; readonly requireMcpOutput: ( result: unknown, operation: string, @@ -207,6 +217,37 @@ const namedEvidence = () => rawResult: { engine_name: "entry", source: "symbol table" }, }, ); +const ghidraEvidence = () => { + const provider = { + id: "ghidra", + name: "Ghidra", + version: SUPPORTED_GHIDRA_VERSION, + }; + return createEvidence( + { + path: "/fixture/ghidra-image.elf", + sha256: "b".repeat(64), + format: "elf", + architecture: "x86_64", + }, + provider, + { + operation: "address_name", + parameters: { address: "0x401000" }, + result: "entry", + rawResult: { + address: "00401000", + symbol: "entry", + source: "Ghidra symbol table", + }, + analysisProfile: createAnalysisProfile(provider, { + loader: "ElfLoader", + language_id: "x86:LE:64:default", + compiler_spec_id: "gcc", + }), + }, + ); +}; const textReply = (value: unknown) => ({ content: [{ type: "text", text: JSON.stringify(value) }], }); @@ -241,6 +282,66 @@ it("admits matching complete text and structured Evidence and rejects either mal ).toThrow(); }); +it.each([false, true])( + "keeps Ghidra contract admission separate from Evidence authentication (structured=%s)", + async (structured) => { + const { requireMcpOutput } = await verifierResults(); + const evidence = ghidraEvidence(); + const reply = (value: unknown) => ({ + ...textReply(value), + ...(structured ? { structuredContent: value } : {}), + }); + const admitted = requireMcpOutput(reply(evidence), "address_name"); + expect(admitted).toEqual(evidence); + const authenticated = parseEvidence(admitted); + expect(authenticated).toEqual(evidence); + expect(authenticated.analysis_profile?.provider).toEqual(evidence.provider); + expect(authenticated.subject).toEqual(evidence.subject); + const tampered = { ...evidence, normalized_result: "different_entry" }; + const shapeValid = requireMcpOutput(reply(tampered), "address_name"); + expect(shapeValid).toEqual(tampered); + expect(() => parseEvidence(shapeValid)).toThrow(/semantic identifier/u); + }, +); + +it("rejects malformed and disagreeing replies at the Ghidra address-context verifier call boundary", async () => { + const source = await readFile( + new URL( + "../../../scripts/verify-real-ghidra-address-context.mjs", + import.meta.url, + ), + "utf8", + ); + const declaration = parse(source, { sourceType: "module" }).program.body.find( + (node) => node.type === "FunctionDeclaration" && node.id?.name === "call", + ); + if (declaration?.start == null || declaration.end == null) + throw new Error("Ghidra verifier call boundary missing"); + const evidence = ghidraEvidence(); + let response: unknown; + // Execute the script's actual call boundary without launching its compiler or provider. + const call = runInNewContext( + `(${source.slice(declaration.start, declaration.end)})`, + { + assert, + client: { callTool: async () => response }, + ...(await verifierResults()), + }, + ) as (name: string) => Promise; + response = { ...textReply(evidence), structuredContent: evidence }; + await expect(call("address_name")).resolves.toEqual(evidence); + const malformed = { ...evidence, normalized_result: { name: "entry" } }; + response = { ...textReply(malformed), structuredContent: malformed }; + await expect(call("address_name")).rejects.toThrow(); + response = { + ...textReply(evidence), + structuredContent: { ...evidence, normalized_result: "different_entry" }, + }; + await expect(call("address_name")).rejects.toThrow( + "text and structured results disagree", + ); +}); + it("rejects disagreement in lifecycle envelope fields that the named schema would otherwise discard", async () => { const { requireMcpOutput } = await verifierResults(); const result = retainedBundle(); @@ -260,7 +361,7 @@ it("rejects disagreement in lifecycle envelope fields that the named schema woul it("rejects disagreement in complete Evidence provenance before downstream extraction", async () => { const { requireMcpOutput } = await verifierResults(); - const evidence = namedEvidence(); + const evidence = ghidraEvidence(); expect(() => requireMcpOutput( {