mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(verifiers): consolidate Ghidra MCP result admission
This commit is contained in:
1 parent
60b55be7e7
commit
3d74c480e0
11 files changed
+164
-56
No files matched your search
@@ -18,7 +18,10 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2)
|
||||
@@ -464,10 +467,9 @@ async function call(name, args = {}) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
if (["open_binary", "binary_session", "close_binary"].includes(name))
|
||||
return requireMcpOperationResult(result, name);
|
||||
return result.structuredContent;
|
||||
return requireMcpOutput(result, name);
|
||||
}
|
||||
|
||||
function evidence(value, sha256) {
|
||||
|
||||
@@ -9,7 +9,10 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2 || process.platform !== "linux")
|
||||
@@ -320,8 +323,7 @@ async function call(name, args = {}) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
return ["open_binary", "binary_session", "close_binary"].includes(name)
|
||||
? requireMcpOperationResult(result, name)
|
||||
: result.structuredContent;
|
||||
: requireMcpOutput(result, name);
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
@@ -339,16 +342,11 @@ async function call(name, args = {}) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(value.isError, true, JSON.stringify(value));
|
||||
const projected = requireMcpOperationResult(value, name);
|
||||
if (name === "open_binary" || name === "close_binary") return projected;
|
||||
const evidence = parseEvidence(value.structuredContent);
|
||||
if (["open_binary", "close_binary"].includes(name))
|
||||
return requireMcpLifecycleResult(value, name);
|
||||
const evidence = parseEvidence(requireMcpOutput(value, name));
|
||||
assert.equal(evidence.subject.digest.sha256, sha256);
|
||||
assert.deepEqual(
|
||||
evidence.normalized_result,
|
||||
value.structuredContent.normalized_result,
|
||||
);
|
||||
return projected;
|
||||
return evidence.normalized_result;
|
||||
}
|
||||
async function cli(command, address, options = []) {
|
||||
const { stdout } = await promisify(execFile)(
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
@@ -555,7 +558,6 @@ async function call(name, arguments_ = {}) {
|
||||
{ name, arguments: arguments_ },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
assert.ok(result.structuredContent, `${name} omitted structured result`);
|
||||
return requireMcpOperationResult(result, name);
|
||||
}
|
||||
@@ -564,8 +566,7 @@ async function evidenceCall(name, arguments_) {
|
||||
{ name, arguments: arguments_ },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
const evidence = parseEvidence(result.structuredContent);
|
||||
const evidence = parseEvidence(requireMcpOutput(result, name));
|
||||
assert.equal(evidence.subject.digest.sha256, sha256);
|
||||
assert.equal(evidence.provider.id, "ghidra");
|
||||
assertProfile(evidence.analysis_profile);
|
||||
|
||||
@@ -16,7 +16,10 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2)
|
||||
@@ -281,10 +284,9 @@ async function call(name, args = {}) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
if (["open_binary", "binary_session", "close_binary"].includes(name))
|
||||
return requireMcpOperationResult(result, name);
|
||||
return result.structuredContent;
|
||||
return requireMcpOutput(result, name);
|
||||
}
|
||||
|
||||
function evidence(value) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
@@ -394,14 +397,8 @@ async function call(name, args = {}) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 360000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
const projected = requireMcpOperationResult(response, name);
|
||||
if (name === "close_binary") return projected;
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
if (name === "close_binary") return requireMcpLifecycleResult(response, name);
|
||||
const evidence = parseEvidence(requireMcpOutput(response, name));
|
||||
assert.equal(evidence.subject.digest.sha256, sha256);
|
||||
assert.deepEqual(
|
||||
evidence.normalized_result,
|
||||
response.structuredContent.normalized_result,
|
||||
);
|
||||
return projected;
|
||||
return evidence.normalized_result;
|
||||
}
|
||||
@@ -12,7 +12,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-inspection.mjs";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2)
|
||||
@@ -351,15 +354,9 @@ async function call(name, args = {}, targetSha256 = sha256) {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
const projected = requireMcpOperationResult(result, name);
|
||||
if (
|
||||
name === "open_binary" ||
|
||||
name === "close_binary" ||
|
||||
name === "binary_session"
|
||||
)
|
||||
return projected;
|
||||
const evidence = parseEvidence(result.structuredContent);
|
||||
if (["open_binary", "close_binary", "binary_session"].includes(name))
|
||||
return requireMcpLifecycleResult(result, name);
|
||||
const evidence = parseEvidence(requireMcpOutput(result, name));
|
||||
assert.equal(evidence.subject.digest.sha256, targetSha256);
|
||||
assert.equal(evidence.subject.format, "raw-binary");
|
||||
return evidence;
|
||||
|
||||
@@ -26,7 +26,8 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins
|
||||
import { buildRiscvFixture } from "../tests/conformance/ghidra/riscv-fixture.mjs";
|
||||
import {
|
||||
mcpTextValue,
|
||||
requireMcpOperationResult,
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
@@ -141,18 +142,24 @@ try {
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
const projected = requireMcpOperationResult(reply, name);
|
||||
const lifecycle = [
|
||||
"open_binary",
|
||||
"close_binary",
|
||||
"binary_session",
|
||||
].includes(name);
|
||||
const admitted = lifecycle
|
||||
? requireMcpLifecycleResult(reply, name)
|
||||
: parseEvidence(requireMcpOutput(reply, name));
|
||||
const textual = JSON.parse(mcpTextValue(reply));
|
||||
assert.deepEqual(reply.structuredContent, textual);
|
||||
assert.ok(reply.structuredContent, `${name} omitted structured result`);
|
||||
const validate = validators.get(name);
|
||||
assert.ok(validate, `${name} missing from advertised catalog`);
|
||||
assert.ok(
|
||||
validate(textual),
|
||||
`${name}: ${JSON.stringify(validate.errors)}`,
|
||||
);
|
||||
if (!["open_binary", "close_binary", "binary_session"].includes(name))
|
||||
assertEvidence(parseEvidence(textual));
|
||||
return projected;
|
||||
if (!lifecycle) assertEvidence(admitted);
|
||||
return lifecycle ? admitted : admitted.normalized_result;
|
||||
};
|
||||
const opened = await call("open_binary", {
|
||||
path: fixture.path,
|
||||
|
||||
@@ -21,7 +21,6 @@ import {
|
||||
ProviderProcessSupervisor,
|
||||
spawnOwnedProviderProcess,
|
||||
} from "../dist/process/ProviderProcess.js";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
@@ -215,7 +214,7 @@ async function call(name, arguments_) {
|
||||
{ timeout: 180_000 },
|
||||
);
|
||||
if (name === "analyze_swift_types")
|
||||
toolContract(name).outputSchema.parse(response.structuredContent);
|
||||
assert.ok(response.structuredContent, `${name} omitted structured result`);
|
||||
return requireMcpOperationResult(response, name);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
@@ -326,9 +329,8 @@ async function toolResult(client, name, arguments_ = {}) {
|
||||
{ name, arguments: arguments_ },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
assert.ok(result.structuredContent, `${name} omitted structured result`);
|
||||
return result.structuredContent;
|
||||
return requireMcpOutput(result, name);
|
||||
}
|
||||
async function call(client, name, arguments_ = {}) {
|
||||
const result = await client.callTool(
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { runInNewContext } from "node:vm";
|
||||
import { parse } from "@babel/parser";
|
||||
import type { ToolContract } from "../../../src/contracts/toolContractTypes.js";
|
||||
import { createEvidence } from "../../../src/domain/evidence.js";
|
||||
import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { SUPPORTED_GHIDRA_VERSION } from "../../../src/ghidra/GhidraReleasePolicy.js";
|
||||
import {
|
||||
createEvidenceBundle,
|
||||
type EvidenceBundle,
|
||||
@@ -8,6 +14,10 @@ import {
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
interface VerifierResults {
|
||||
readonly requireMcpOperationResult: (
|
||||
result: unknown,
|
||||
operation: string,
|
||||
) => unknown;
|
||||
readonly requireMcpOutput: (
|
||||
result: unknown,
|
||||
operation: string,
|
||||
@@ -207,6 +217,37 @@ const namedEvidence = () =>
|
||||
rawResult: { engine_name: "entry", source: "symbol table" },
|
||||
},
|
||||
);
|
||||
const ghidraEvidence = () => {
|
||||
const provider = {
|
||||
id: "ghidra",
|
||||
name: "Ghidra",
|
||||
version: SUPPORTED_GHIDRA_VERSION,
|
||||
};
|
||||
return createEvidence(
|
||||
{
|
||||
path: "/fixture/ghidra-image.elf",
|
||||
sha256: "b".repeat(64),
|
||||
format: "elf",
|
||||
architecture: "x86_64",
|
||||
},
|
||||
provider,
|
||||
{
|
||||
operation: "address_name",
|
||||
parameters: { address: "0x401000" },
|
||||
result: "entry",
|
||||
rawResult: {
|
||||
address: "00401000",
|
||||
symbol: "entry",
|
||||
source: "Ghidra symbol table",
|
||||
},
|
||||
analysisProfile: createAnalysisProfile(provider, {
|
||||
loader: "ElfLoader",
|
||||
language_id: "x86:LE:64:default",
|
||||
compiler_spec_id: "gcc",
|
||||
}),
|
||||
},
|
||||
);
|
||||
};
|
||||
const textReply = (value: unknown) => ({
|
||||
content: [{ type: "text", text: JSON.stringify(value) }],
|
||||
});
|
||||
@@ -241,6 +282,66 @@ it("admits matching complete text and structured Evidence and rejects either mal
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"keeps Ghidra contract admission separate from Evidence authentication (structured=%s)",
|
||||
async (structured) => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const evidence = ghidraEvidence();
|
||||
const reply = (value: unknown) => ({
|
||||
...textReply(value),
|
||||
...(structured ? { structuredContent: value } : {}),
|
||||
});
|
||||
const admitted = requireMcpOutput(reply(evidence), "address_name");
|
||||
expect(admitted).toEqual(evidence);
|
||||
const authenticated = parseEvidence(admitted);
|
||||
expect(authenticated).toEqual(evidence);
|
||||
expect(authenticated.analysis_profile?.provider).toEqual(evidence.provider);
|
||||
expect(authenticated.subject).toEqual(evidence.subject);
|
||||
const tampered = { ...evidence, normalized_result: "different_entry" };
|
||||
const shapeValid = requireMcpOutput(reply(tampered), "address_name");
|
||||
expect(shapeValid).toEqual(tampered);
|
||||
expect(() => parseEvidence(shapeValid)).toThrow(/semantic identifier/u);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects malformed and disagreeing replies at the Ghidra address-context verifier call boundary", async () => {
|
||||
const source = await readFile(
|
||||
new URL(
|
||||
"../../../scripts/verify-real-ghidra-address-context.mjs",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
const declaration = parse(source, { sourceType: "module" }).program.body.find(
|
||||
(node) => node.type === "FunctionDeclaration" && node.id?.name === "call",
|
||||
);
|
||||
if (declaration?.start == null || declaration.end == null)
|
||||
throw new Error("Ghidra verifier call boundary missing");
|
||||
const evidence = ghidraEvidence();
|
||||
let response: unknown;
|
||||
// Execute the script's actual call boundary without launching its compiler or provider.
|
||||
const call = runInNewContext(
|
||||
`(${source.slice(declaration.start, declaration.end)})`,
|
||||
{
|
||||
assert,
|
||||
client: { callTool: async () => response },
|
||||
...(await verifierResults()),
|
||||
},
|
||||
) as (name: string) => Promise<unknown>;
|
||||
response = { ...textReply(evidence), structuredContent: evidence };
|
||||
await expect(call("address_name")).resolves.toEqual(evidence);
|
||||
const malformed = { ...evidence, normalized_result: { name: "entry" } };
|
||||
response = { ...textReply(malformed), structuredContent: malformed };
|
||||
await expect(call("address_name")).rejects.toThrow();
|
||||
response = {
|
||||
...textReply(evidence),
|
||||
structuredContent: { ...evidence, normalized_result: "different_entry" },
|
||||
};
|
||||
await expect(call("address_name")).rejects.toThrow(
|
||||
"text and structured results disagree",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects disagreement in lifecycle envelope fields that the named schema would otherwise discard", async () => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const result = retainedBundle();
|
||||
@@ -260,7 +361,7 @@ it("rejects disagreement in lifecycle envelope fields that the named schema woul
|
||||
|
||||
it("rejects disagreement in complete Evidence provenance before downstream extraction", async () => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const evidence = namedEvidence();
|
||||
const evidence = ghidraEvidence();
|
||||
expect(() =>
|
||||
requireMcpOutput(
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user