mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
refactor(verifiers): centralize named artifact output admission
Route six artifact verifiers through the canonical complete MCP output and error parsers. Remove duplicated JSON/schema interpretation while retaining Evidence authentication, provider provenance, independent CLI parsing and CLI/MCP parity checks. Validate malformed success/error envelopes at the shared boundary. Validation: 11 boundary tests, Go compiler verifier (30 cases), and synthetic IL2CPP/Flutter verifier workflows passed; typecheck/lint passed. PCAP readiness passed with stub tools, but capture inspection is unsupported on macOS. Linux-only binary-layout and EVM verifier toolchains were unavailable.
This commit is contained in:
1 parent
f02df45a46
commit
0828917c79
8 files changed
+147
-65
No files matched your search
@@ -28,7 +28,12 @@ export const requireMcpToolError = (result) => {
|
||||
return analysisErrorProjectionSchema.parse(jsonValue(result).error);
|
||||
};
|
||||
|
||||
const requireOutput = (result, operation, contract) => {
|
||||
/** Validate the complete output against its exact named contract, retaining inline Evidence. */
|
||||
export const requireMcpOutput = (
|
||||
result,
|
||||
operation,
|
||||
contract = toolContract(operation),
|
||||
) => {
|
||||
if (result.isError === true)
|
||||
throw new Error(`${operation} failed: ${mcpTextValue(result)}`);
|
||||
if (contract.name !== operation)
|
||||
@@ -96,7 +101,7 @@ export const requireMcpEvidenceResult = (result, operation) => {
|
||||
const contract = toolContract(operation);
|
||||
if (operationResultKey(contract) !== "normalized_result")
|
||||
throw new Error(`${operation} requires the lifecycle result extractor`);
|
||||
return requireOutput(result, operation, contract).normalized_result;
|
||||
return requireMcpOutput(result, operation, contract).normalized_result;
|
||||
};
|
||||
|
||||
/** Validate a lifecycle tool output and return its operation result. */
|
||||
@@ -104,7 +109,7 @@ export const requireMcpLifecycleResult = (result, operation) => {
|
||||
const contract = toolContract(operation);
|
||||
if (operationResultKey(contract) !== "result")
|
||||
throw new Error(`${operation} requires the Evidence result extractor`);
|
||||
const projected = requireOutput(result, operation, contract).result;
|
||||
const projected = requireMcpOutput(result, operation, contract).result;
|
||||
return operation === "binary_session"
|
||||
? expandMcpSessionStatus(projected)
|
||||
: projected;
|
||||
@@ -114,7 +119,7 @@ export const requireMcpLifecycleResult = (result, operation) => {
|
||||
export const requireMcpEvidenceBundle = (result) => {
|
||||
const operation = "get_evidence_bundle";
|
||||
const contract = toolContract(operation);
|
||||
const bundle = requireOutput(result, operation, contract).result;
|
||||
const bundle = requireMcpOutput(result, operation, contract).result;
|
||||
if (bundle.kind === "evidence-bundle-summary")
|
||||
throw new Error(
|
||||
"get_evidence_bundle returned summary metadata instead of complete Evidence",
|
||||
@@ -138,7 +143,7 @@ export const requireMcpOperationResult = (
|
||||
contract = toolContract(operation),
|
||||
) => {
|
||||
const key = operationResultKey(contract);
|
||||
const projected = requireOutput(result, operation, contract)[key];
|
||||
const projected = requireMcpOutput(result, operation, contract)[key];
|
||||
return operation === "binary_session"
|
||||
? expandMcpSessionStatus(projected)
|
||||
: projected;
|
||||
|
||||
@@ -20,7 +20,10 @@ import {
|
||||
runOwnedCommand,
|
||||
} from "../dist/process/OwnedCommand.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { sectionNameFixtures } from "./lib/elf-section-name-fixtures.mjs";
|
||||
import { withLargeResultMcp } from "./lib/large-result-mcp.mjs";
|
||||
@@ -1549,14 +1552,12 @@ async function inspect(
|
||||
},
|
||||
{ timeout: 300_000 },
|
||||
);
|
||||
const value = JSON.parse(mcpTextValue(response));
|
||||
if (category !== undefined) {
|
||||
assert.equal(response.isError, true);
|
||||
assert.equal(value.error.category, category, JSON.stringify(value.error));
|
||||
return value.error;
|
||||
const error = requireMcpToolError(response);
|
||||
assert.equal(error.category, category, JSON.stringify(error));
|
||||
return error;
|
||||
}
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
envelope = value;
|
||||
envelope = requireMcpOutput(response, "inspect_binary_layout");
|
||||
} else {
|
||||
let response;
|
||||
try {
|
||||
|
||||
@@ -11,7 +11,10 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const solcModule = process.env.REA_VERIFY_SOLC_MODULE;
|
||||
@@ -103,11 +106,10 @@ try {
|
||||
name: "inspect_evm_interface",
|
||||
arguments: { path, encoding },
|
||||
});
|
||||
value = JSON.parse(mcpTextValue(reply));
|
||||
if (expectedCategory !== undefined) {
|
||||
assert.equal(reply.isError, true);
|
||||
value = value.error;
|
||||
} else assert.notEqual(reply.isError, true, mcpTextValue(reply));
|
||||
value =
|
||||
expectedCategory !== undefined
|
||||
? requireMcpToolError(reply)
|
||||
: requireMcpOutput(reply, "inspect_evm_interface");
|
||||
}
|
||||
if (expectedCategory !== undefined) {
|
||||
assert.equal(value.category, expectedCategory, JSON.stringify(value));
|
||||
|
||||
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
closeVerifierResources,
|
||||
withVerifierCleanup,
|
||||
@@ -234,8 +234,7 @@ async function run(operation, input) {
|
||||
{ name: operation, arguments: input },
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
|
||||
const mcp = requireMcpOutput(response, operation, contract);
|
||||
assert.deepEqual(cli.normalized_result, mcp.normalized_result);
|
||||
assert.deepEqual(cli.subject, mcp.subject);
|
||||
assert.equal(cli.subject.local_path, input.path);
|
||||
|
||||
@@ -12,7 +12,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { goBinarySchema } from "../dist/domain/go/goBinary.js";
|
||||
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
@@ -359,8 +359,7 @@ async function inspect(mode, path) {
|
||||
{ name: "inspect_go_binary", arguments: { path } },
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
envelope = JSON.parse(mcpTextValue(response));
|
||||
envelope = requireMcpOutput(response, "inspect_go_binary");
|
||||
}
|
||||
const evidence = parseEvidence(envelope);
|
||||
assert.equal(evidence.subject.local_path, path);
|
||||
|
||||
@@ -6,9 +6,11 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { absoluteCallerPath } from "../dist/filesystem/callerPath.js";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { analysisViewResultSchema } from "../dist/domain/analysisView/analysisView.js";
|
||||
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
import { hashVerifierFile } from "./lib/verifier-file-resources.mjs";
|
||||
|
||||
@@ -43,8 +45,7 @@ const call = async (name, input) => {
|
||||
{ name, arguments: input },
|
||||
{ timeout: 120_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
return JSON.parse(mcpTextValue(response));
|
||||
return requireMcpOutput(response, name);
|
||||
};
|
||||
const resources = new ArtifactResourceScope();
|
||||
await withVerifierCleanup(async () => {
|
||||
@@ -74,7 +75,7 @@ await withVerifierCleanup(async () => {
|
||||
if (complete.isError === true) {
|
||||
// Large inventories exceed the stdio budget; the result must stay
|
||||
// retained and addressable rather than truncated.
|
||||
const { error } = JSON.parse(mcpTextValue(complete));
|
||||
const error = requireMcpToolError(complete);
|
||||
assert.equal(error.code, "resource_constraint");
|
||||
assert.equal(error.details.reported_limits.boundary, "mcp-response");
|
||||
assert.match(error.details.reported_limits.evidence_id, /^ev_/u);
|
||||
@@ -82,9 +83,7 @@ await withVerifierCleanup(async () => {
|
||||
`[ok] MCP complete delivery refused at the stdio budget (${error.details.reported_limits.response_bytes_at_least} bytes) with retained Evidence`,
|
||||
);
|
||||
} else {
|
||||
const mcp = contract.outputSchema.parse(
|
||||
JSON.parse(mcpTextValue(complete)),
|
||||
);
|
||||
const mcp = requireMcpOutput(complete, operation, contract);
|
||||
assert.deepEqual(mcp.normalized_result, report);
|
||||
console.log("[ok] CLI/MCP complete Evidence parity");
|
||||
}
|
||||
@@ -108,9 +107,7 @@ await withVerifierCleanup(async () => {
|
||||
});
|
||||
console.log("[ok] image type ranges and type method ranges agree");
|
||||
|
||||
const summary = contract.outputSchema.parse(
|
||||
await call(operation, { path }),
|
||||
).normalized_result;
|
||||
const summary = (await call(operation, { path })).normalized_result;
|
||||
assert.equal(summary.summary.magic, report.magic);
|
||||
assert.deepEqual(summary.summary.counts, {
|
||||
sections: report.sections.length,
|
||||
@@ -135,37 +132,33 @@ await withVerifierCleanup(async () => {
|
||||
generic_method_instance_states: null,
|
||||
generic_method_issues: null,
|
||||
});
|
||||
const page = analysisViewResultSchema.parse(
|
||||
(
|
||||
await call("inspect_analysis_view", {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
},
|
||||
view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 },
|
||||
})
|
||||
).normalized_result,
|
||||
);
|
||||
const page = (
|
||||
await call("inspect_analysis_view", {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
},
|
||||
view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 },
|
||||
})
|
||||
).normalized_result;
|
||||
assert.deepEqual(
|
||||
page.items,
|
||||
report.methods.slice(0, 16).map((row, index) => ({ ...row, index })),
|
||||
);
|
||||
const last = analysisViewResultSchema.parse(
|
||||
(
|
||||
await call("inspect_analysis_view", {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
},
|
||||
view: {
|
||||
kind: "il2cpp-metadata",
|
||||
collection: "types",
|
||||
offset: Math.max(0, report.types.length - 4),
|
||||
limit: 16,
|
||||
},
|
||||
})
|
||||
).normalized_result,
|
||||
);
|
||||
const last = (
|
||||
await call("inspect_analysis_view", {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
},
|
||||
view: {
|
||||
kind: "il2cpp-metadata",
|
||||
collection: "types",
|
||||
offset: Math.max(0, report.types.length - 4),
|
||||
limit: 16,
|
||||
},
|
||||
})
|
||||
).normalized_result;
|
||||
const lastOffset = Math.max(0, report.types.length - 4);
|
||||
assert.deepEqual(
|
||||
last.items,
|
||||
|
||||
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { ArtifactResourceScope } from "../dist/artifacts/ArtifactResourceScope.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
closeVerifierResources,
|
||||
withVerifierCleanup,
|
||||
@@ -123,8 +123,7 @@ async function run(name, input) {
|
||||
{ name, arguments: input },
|
||||
{ timeout: 360_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
|
||||
const mcp = requireMcpOutput(response, name, contract);
|
||||
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
|
||||
assert.deepEqual(mcp.subject, cli.subject);
|
||||
if (input.path !== undefined) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
import type { ToolContract } from "../../../src/contracts/toolContractTypes.js";
|
||||
import { createEvidence } from "../../../src/domain/evidence.js";
|
||||
import {
|
||||
createEvidenceBundle,
|
||||
@@ -6,6 +8,12 @@ import {
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
interface VerifierResults {
|
||||
readonly requireMcpOutput: (
|
||||
result: unknown,
|
||||
operation: string,
|
||||
contract?: ToolContract,
|
||||
) => unknown;
|
||||
readonly requireMcpToolError: (result: unknown) => unknown;
|
||||
readonly requireMcpEvidenceBundle: (result: unknown) => EvidenceBundle;
|
||||
readonly requireMcpLifecycleResult: (
|
||||
result: unknown,
|
||||
@@ -182,3 +190,79 @@ it("retains summary discovery in the generic lifecycle reader but refuses it as
|
||||
"summary metadata instead of complete Evidence",
|
||||
);
|
||||
});
|
||||
|
||||
const namedEvidence = () =>
|
||||
createEvidence(
|
||||
{
|
||||
path: "/fixture/image.bin",
|
||||
sha256: "a".repeat(64),
|
||||
format: "elf",
|
||||
architecture: "x86_64",
|
||||
},
|
||||
{ id: "fixture", name: "Fixture", version: "1" },
|
||||
{
|
||||
operation: "address_name",
|
||||
parameters: { address: "0x1000" },
|
||||
result: "entry",
|
||||
rawResult: { engine_name: "entry", source: "symbol table" },
|
||||
},
|
||||
);
|
||||
const textReply = (value: unknown) => ({
|
||||
content: [{ type: "text", text: JSON.stringify(value) }],
|
||||
});
|
||||
|
||||
it("returns complete named Evidence with its subject, provider and analysis provenance", async () => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const evidence = namedEvidence();
|
||||
expect(requireMcpOutput(textReply(evidence), "address_name")).toEqual(
|
||||
evidence,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects mismatched named contracts, error-marked success and malformed normalized results", async () => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const evidence = namedEvidence();
|
||||
expect(() =>
|
||||
requireMcpOutput(
|
||||
textReply(evidence),
|
||||
"address_name",
|
||||
toolContract("set_address_name"),
|
||||
),
|
||||
).toThrow("received the set_address_name contract");
|
||||
expect(() =>
|
||||
requireMcpOutput({ ...textReply(evidence), isError: true }, "address_name"),
|
||||
).toThrow("address_name failed");
|
||||
expect(() =>
|
||||
requireMcpOutput(
|
||||
textReply({ ...evidence, normalized_result: { name: "entry" } }),
|
||||
"address_name",
|
||||
),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("requires canonical complete tool errors before inspecting their diagnostic fields", async () => {
|
||||
const { requireMcpToolError } = await verifierResults();
|
||||
const error = {
|
||||
code: "invalid_request",
|
||||
category: "invalid_input",
|
||||
message: "The address is invalid",
|
||||
retryable: false,
|
||||
remediation: { action: "Choose an observed address" },
|
||||
details: { operation: "address_name", address: "bad" },
|
||||
};
|
||||
expect(
|
||||
requireMcpToolError({ ...textReply({ error }), isError: true }),
|
||||
).toEqual(error);
|
||||
for (const value of [
|
||||
{ error: { category: "invalid_input" } },
|
||||
error,
|
||||
{ error: { ...error, retryable: "false" } },
|
||||
{ error: { ...error, code: "vendor_failure" } },
|
||||
])
|
||||
expect(() =>
|
||||
requireMcpToolError({ ...textReply(value), isError: true }),
|
||||
).toThrow();
|
||||
expect(() => requireMcpToolError(textReply({ error }))).toThrow(
|
||||
"Expected an MCP tool failure",
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user