refactor(verifiers): centralize named artifact output admission

Route six artifact verifiers through the canonical complete MCP output and error parsers. Remove duplicated JSON/schema interpretation while retaining Evidence authentication, provider provenance, independent CLI parsing and CLI/MCP parity checks. Validate malformed success/error envelopes at the shared boundary.

Validation: 11 boundary tests, Go compiler verifier (30 cases), and synthetic IL2CPP/Flutter verifier workflows passed; typecheck/lint passed. PCAP readiness passed with stub tools, but capture inspection is unsupported on macOS. Linux-only binary-layout and EVM verifier toolchains were unavailable.
This commit is contained in:
morluto committed 2026-10-26 22:48:22 +08:00
1 parent f02df45a46
commit 0828917c79
8 files changed
+147 -65

No files matched your search

+10 -5
View File
@@ -28,7 +28,12 @@ export const requireMcpToolError = (result) => {
return analysisErrorProjectionSchema.parse(jsonValue(result).error);
};
const requireOutput = (result, operation, contract) => {
/** Validate the complete output against its exact named contract, retaining inline Evidence. */
export const requireMcpOutput = (
result,
operation,
contract = toolContract(operation),
) => {
if (result.isError === true)
throw new Error(`${operation} failed: ${mcpTextValue(result)}`);
if (contract.name !== operation)
@@ -96,7 +101,7 @@ export const requireMcpEvidenceResult = (result, operation) => {
const contract = toolContract(operation);
if (operationResultKey(contract) !== "normalized_result")
throw new Error(`${operation} requires the lifecycle result extractor`);
return requireOutput(result, operation, contract).normalized_result;
return requireMcpOutput(result, operation, contract).normalized_result;
};
/** Validate a lifecycle tool output and return its operation result. */
@@ -104,7 +109,7 @@ export const requireMcpLifecycleResult = (result, operation) => {
const contract = toolContract(operation);
if (operationResultKey(contract) !== "result")
throw new Error(`${operation} requires the Evidence result extractor`);
const projected = requireOutput(result, operation, contract).result;
const projected = requireMcpOutput(result, operation, contract).result;
return operation === "binary_session"
? expandMcpSessionStatus(projected)
: projected;
@@ -114,7 +119,7 @@ export const requireMcpLifecycleResult = (result, operation) => {
export const requireMcpEvidenceBundle = (result) => {
const operation = "get_evidence_bundle";
const contract = toolContract(operation);
const bundle = requireOutput(result, operation, contract).result;
const bundle = requireMcpOutput(result, operation, contract).result;
if (bundle.kind === "evidence-bundle-summary")
throw new Error(
"get_evidence_bundle returned summary metadata instead of complete Evidence",
@@ -138,7 +143,7 @@ export const requireMcpOperationResult = (
contract = toolContract(operation),
) => {
const key = operationResultKey(contract);
const projected = requireOutput(result, operation, contract)[key];
const projected = requireMcpOutput(result, operation, contract)[key];
return operation === "binary_session"
? expandMcpSessionStatus(projected)
: projected;
+8 -7
View File
@@ -20,7 +20,10 @@ import {
runOwnedCommand,
} from "../dist/process/OwnedCommand.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import {
requireMcpOutput,
requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { sectionNameFixtures } from "./lib/elf-section-name-fixtures.mjs";
import { withLargeResultMcp } from "./lib/large-result-mcp.mjs";
@@ -1549,14 +1552,12 @@ async function inspect(
},
{ timeout: 300_000 },
);
const value = JSON.parse(mcpTextValue(response));
if (category !== undefined) {
assert.equal(response.isError, true);
assert.equal(value.error.category, category, JSON.stringify(value.error));
return value.error;
const error = requireMcpToolError(response);
assert.equal(error.category, category, JSON.stringify(error));
return error;
}
assert.notEqual(response.isError, true, mcpTextValue(response));
envelope = value;
envelope = requireMcpOutput(response, "inspect_binary_layout");
} else {
let response;
try {
+8 -6
View File
@@ -11,7 +11,10 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import {
requireMcpOutput,
requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
const solcModule = process.env.REA_VERIFY_SOLC_MODULE;
@@ -103,11 +106,10 @@ try {
name: "inspect_evm_interface",
arguments: { path, encoding },
});
value = JSON.parse(mcpTextValue(reply));
if (expectedCategory !== undefined) {
assert.equal(reply.isError, true);
value = value.error;
} else assert.notEqual(reply.isError, true, mcpTextValue(reply));
value =
expectedCategory !== undefined
? requireMcpToolError(reply)
: requireMcpOutput(reply, "inspect_evm_interface");
}
if (expectedCategory !== undefined) {
assert.equal(value.category, expectedCategory, JSON.stringify(value));
+2 -3
View File
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
@@ -234,8 +234,7 @@ async function run(operation, input) {
{ name: operation, arguments: input },
{ timeout: 60_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
const mcp = requireMcpOutput(response, operation, contract);
assert.deepEqual(cli.normalized_result, mcp.normalized_result);
assert.deepEqual(cli.subject, mcp.subject);
assert.equal(cli.subject.local_path, input.path);
+2 -3
View File
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../dist/domain/evidence.js";
import { goBinarySchema } from "../dist/domain/go/goBinary.js";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
const execute = promisify(execFile);
@@ -359,8 +359,7 @@ async function inspect(mode, path) {
{ name: "inspect_go_binary", arguments: { path } },
{ timeout: 30_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
envelope = JSON.parse(mcpTextValue(response));
envelope = requireMcpOutput(response, "inspect_go_binary");
}
const evidence = parseEvidence(envelope);
assert.equal(evidence.subject.local_path, path);
+31 -38
View File
@@ -6,9 +6,11 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { absoluteCallerPath } from "../dist/filesystem/callerPath.js";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { analysisViewResultSchema } from "../dist/domain/analysisView/analysisView.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import {
requireMcpOutput,
requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
@@ -43,8 +45,7 @@ const call = async (name, input) => {
{ name, arguments: input },
{ timeout: 120_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
return JSON.parse(mcpTextValue(response));
return requireMcpOutput(response, name);
};
const resources = new ArtifactResourceScope();
await withVerifierCleanup(async () => {
@@ -74,7 +75,7 @@ await withVerifierCleanup(async () => {
if (complete.isError === true) {
// Large inventories exceed the stdio budget; the result must stay
// retained and addressable rather than truncated.
const { error } = JSON.parse(mcpTextValue(complete));
const error = requireMcpToolError(complete);
assert.equal(error.code, "resource_constraint");
assert.equal(error.details.reported_limits.boundary, "mcp-response");
assert.match(error.details.reported_limits.evidence_id, /^ev_/u);
@@ -82,9 +83,7 @@ await withVerifierCleanup(async () => {
`[ok] MCP complete delivery refused at the stdio budget (${error.details.reported_limits.response_bytes_at_least} bytes) with retained Evidence`,
);
} else {
const mcp = contract.outputSchema.parse(
JSON.parse(mcpTextValue(complete)),
);
const mcp = requireMcpOutput(complete, operation, contract);
assert.deepEqual(mcp.normalized_result, report);
console.log("[ok] CLI/MCP complete Evidence parity");
}
@@ -108,9 +107,7 @@ await withVerifierCleanup(async () => {
});
console.log("[ok] image type ranges and type method ranges agree");
const summary = contract.outputSchema.parse(
await call(operation, { path }),
).normalized_result;
const summary = (await call(operation, { path })).normalized_result;
assert.equal(summary.summary.magic, report.magic);
assert.deepEqual(summary.summary.counts, {
sections: report.sections.length,
@@ -135,37 +132,33 @@ await withVerifierCleanup(async () => {
generic_method_instance_states: null,
generic_method_issues: null,
});
const page = analysisViewResultSchema.parse(
(
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
},
view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 },
})
).normalized_result,
);
const page = (
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
},
view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 },
})
).normalized_result;
assert.deepEqual(
page.items,
report.methods.slice(0, 16).map((row, index) => ({ ...row, index })),
);
const last = analysisViewResultSchema.parse(
(
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
},
view: {
kind: "il2cpp-metadata",
collection: "types",
offset: Math.max(0, report.types.length - 4),
limit: 16,
},
})
).normalized_result,
);
const last = (
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
},
view: {
kind: "il2cpp-metadata",
collection: "types",
offset: Math.max(0, report.types.length - 4),
limit: 16,
},
})
).normalized_result;
const lastOffset = Math.max(0, report.types.length - 4);
assert.deepEqual(
last.items,
+2 -3
View File
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
import {
closeVerifierResources,
withVerifierCleanup,
@@ -123,8 +123,7 @@ async function run(name, input) {
{ name, arguments: input },
{ timeout: 360_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
const mcp = requireMcpOutput(response, name, contract);
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
assert.deepEqual(mcp.subject, cli.subject);
if (input.path !== undefined) {
@@ -1,3 +1,5 @@
import { toolContract } from "../../../src/contracts/toolContracts.js";
import type { ToolContract } from "../../../src/contracts/toolContractTypes.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import {
createEvidenceBundle,
@@ -6,6 +8,12 @@ import {
import { expect, it } from "vitest";
interface VerifierResults {
readonly requireMcpOutput: (
result: unknown,
operation: string,
contract?: ToolContract,
) => unknown;
readonly requireMcpToolError: (result: unknown) => unknown;
readonly requireMcpEvidenceBundle: (result: unknown) => EvidenceBundle;
readonly requireMcpLifecycleResult: (
result: unknown,
@@ -182,3 +190,79 @@ it("retains summary discovery in the generic lifecycle reader but refuses it as
"summary metadata instead of complete Evidence",
);
});
const namedEvidence = () =>
createEvidence(
{
path: "/fixture/image.bin",
sha256: "a".repeat(64),
format: "elf",
architecture: "x86_64",
},
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "address_name",
parameters: { address: "0x1000" },
result: "entry",
rawResult: { engine_name: "entry", source: "symbol table" },
},
);
const textReply = (value: unknown) => ({
content: [{ type: "text", text: JSON.stringify(value) }],
});
it("returns complete named Evidence with its subject, provider and analysis provenance", async () => {
const { requireMcpOutput } = await verifierResults();
const evidence = namedEvidence();
expect(requireMcpOutput(textReply(evidence), "address_name")).toEqual(
evidence,
);
});
it("rejects mismatched named contracts, error-marked success and malformed normalized results", async () => {
const { requireMcpOutput } = await verifierResults();
const evidence = namedEvidence();
expect(() =>
requireMcpOutput(
textReply(evidence),
"address_name",
toolContract("set_address_name"),
),
).toThrow("received the set_address_name contract");
expect(() =>
requireMcpOutput({ ...textReply(evidence), isError: true }, "address_name"),
).toThrow("address_name failed");
expect(() =>
requireMcpOutput(
textReply({ ...evidence, normalized_result: { name: "entry" } }),
"address_name",
),
).toThrow();
});
it("requires canonical complete tool errors before inspecting their diagnostic fields", async () => {
const { requireMcpToolError } = await verifierResults();
const error = {
code: "invalid_request",
category: "invalid_input",
message: "The address is invalid",
retryable: false,
remediation: { action: "Choose an observed address" },
details: { operation: "address_name", address: "bad" },
};
expect(
requireMcpToolError({ ...textReply({ error }), isError: true }),
).toEqual(error);
for (const value of [
{ error: { category: "invalid_input" } },
error,
{ error: { ...error, retryable: "false" } },
{ error: { ...error, code: "vendor_failure" } },
])
expect(() =>
requireMcpToolError({ ...textReply(value), isError: true }),
).toThrow();
expect(() => requireMcpToolError(textReply({ error }))).toThrow(
"Expected an MCP tool failure",
);
});