fix(verifiers): preserve source Evidence through shared admission

This commit is contained in:
morluto committed 2026-10-27 01:11:51 +08:00
1 parent f6805e29c3
commit 662ec2e450
14 files changed
+255 -86

No files matched your search

+11 -6
View File
@@ -1,3 +1,4 @@
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { execFile } from "node:child_process"; import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url"; import { fileURLToPath } from "node:url";
@@ -116,9 +117,12 @@ export async function verifyBrowserNetworkEvidence(
{ timeout: 60_000 }, { timeout: 60_000 },
); );
assert.notEqual(result.isError, true, JSON.stringify(result)); assert.notEqual(result.isError, true, JSON.stringify(result));
const evidence = parseEvidence(result.structuredContent); assert.notEqual(result.structuredContent, undefined);
const evidence = parseEvidence(
requireMcpOutput(result, "capture_browser_scenario"),
);
mcpProof = assertBrowserNetworkEvidence( mcpProof = assertBrowserNetworkEvidence(
browserScenarioCaptureSchema.parse(evidence.normalized_result), evidence.normalized_result,
site.origin, site.origin,
); );
if (attachment !== undefined) { if (attachment !== undefined) {
@@ -131,12 +135,13 @@ export async function verifyBrowserNetworkEvidence(
{ timeout: 60_000 }, { timeout: 60_000 },
); );
assert.notEqual(connected.isError, true, JSON.stringify(connected)); assert.notEqual(connected.isError, true, JSON.stringify(connected));
const connectedEvidence = parseEvidence(connected.structuredContent); assert.notEqual(connected.structuredContent, undefined);
const connectedEvidence = parseEvidence(
requireMcpOutput(connected, "capture_browser_scenario"),
);
assert.deepEqual( assert.deepEqual(
assertBrowserNetworkEvidence( assertBrowserNetworkEvidence(
browserScenarioCaptureSchema.parse( connectedEvidence.normalized_result,
connectedEvidence.normalized_result,
),
site.origin, site.origin,
"disconnected-external", "disconnected-external",
), ),
+5 -4
View File
@@ -1,3 +1,4 @@
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { execFile } from "node:child_process"; import { execFile } from "node:child_process";
import { once } from "node:events"; import { once } from "node:events";
@@ -163,11 +164,11 @@ const verifyTransports = async (executable, origin) => {
{ timeout: 60_000 }, { timeout: 60_000 },
); );
assert.notEqual(captured.isError, true, JSON.stringify(captured)); assert.notEqual(captured.isError, true, JSON.stringify(captured));
const evidence = parseEvidence(captured.structuredContent); assert.notEqual(captured.structuredContent, undefined);
assertNetwork( const evidence = parseEvidence(
browserScenarioCaptureSchema.parse(evidence.normalized_result), requireMcpOutput(captured, "capture_browser_scenario"),
origin,
); );
assertNetwork(evidence.normalized_result, origin);
} finally { } finally {
try { try {
await client.close(); await client.close();
+8 -4
View File
@@ -1,4 +1,7 @@
import { requireMcpToolError } from "./mcp-verifier-results.mjs"; import {
requireMcpOutput,
requireMcpToolError,
} from "./mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { execFile } from "node:child_process"; import { execFile } from "node:child_process";
import { mkdtemp, readFile, rm } from "node:fs/promises"; import { mkdtemp, readFile, rm } from "node:fs/promises";
@@ -95,10 +98,11 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
{ timeout: 60000 }, { timeout: 60000 },
); );
assert.notEqual(captured.isError, true, JSON.stringify(captured)); assert.notEqual(captured.isError, true, JSON.stringify(captured));
const mcpEvidence = parseEvidence(captured.structuredContent); assert.notEqual(captured.structuredContent, undefined);
const mcpScreenshot = webScreenshotSchema.parse( const mcpEvidence = parseEvidence(
mcpEvidence.normalized_result, requireMcpOutput(captured, "capture_web_screenshot"),
); );
const mcpScreenshot = mcpEvidence.normalized_result;
assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact); assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact);
const compared = comparePngScreenshots({ const compared = comparePngScreenshots({
before: screenshot.artifact, before: screenshot.artifact,
+2 -1
View File
@@ -46,7 +46,8 @@ export const requireMcpOutput = (
if (!isDeepStrictEqual(text, result.structuredContent)) if (!isDeepStrictEqual(text, result.structuredContent))
throw new Error(`${operation} text and structured results disagree`); throw new Error(`${operation} text and structured results disagree`);
} }
return output; // Downstream parseEvidence must authenticate source facts, not a schema projection.
return Object.hasOwn(output, "normalized_result") ? text : output;
}; };
const operationResultKey = (contract) => { const operationResultKey = (contract) => {
+27 -16
View File
@@ -14,6 +14,7 @@ import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrac
import { import {
requireMcpEvidenceBundle, requireMcpEvidenceBundle,
requireMcpLifecycleResult, requireMcpLifecycleResult,
requireMcpOutput,
} from "./mcp-verifier-results.mjs"; } from "./mcp-verifier-results.mjs";
import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs"; import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs";
@@ -198,12 +199,15 @@ async function verifyNativeValueAdapters(
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(result.isError, true, JSON.stringify(result)); assert.notEqual(result.isError, true, JSON.stringify(result));
const mcpEvidence = parseEvidence(result.structuredContent); assert.notEqual(result.structuredContent, undefined);
const mcpEvidence = parseEvidence(
requireMcpOutput(result, "trace_native_values"),
);
assert.equal(mcpEvidence.operation, "trace_native_values"); assert.equal(mcpEvidence.operation, "trace_native_values");
assert.equal(mcpEvidence.provider.id, "rea-workflow"); assert.equal(mcpEvidence.provider.id, "rea-workflow");
assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile); assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile);
const mcpGraph = verifyGraph( const mcpGraph = verifyGraph(
result.structuredContent?.normalized_result, mcpEvidence.normalized_result,
target, target,
procedure, procedure,
globalAddress, globalAddress,
@@ -218,7 +222,10 @@ async function verifyNativeValueAdapters(
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed)); assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
const functionEvidence = parseEvidence(analyzed.structuredContent); assert.notEqual(analyzed.structuredContent, undefined);
const functionEvidence = parseEvidence(
requireMcpOutput(analyzed, "analyze_function"),
);
assert.equal(functionEvidence.operation, "analyze_function"); assert.equal(functionEvidence.operation, "analyze_function");
assert.equal(functionEvidence.provider.id, "ghidra"); assert.equal(functionEvidence.provider.id, "ghidra");
assert.deepEqual( assert.deepEqual(
@@ -246,10 +253,6 @@ async function verifyNativeValueAdapters(
functionEvidence.analysis_profile, functionEvidence.analysis_profile,
); );
assert.deepEqual(functionProfile, upstreamProfile); assert.deepEqual(functionProfile, upstreamProfile);
assert.deepEqual(
functionEvidence.normalized_result,
analyzed.structuredContent?.normalized_result,
);
const comparison = await client.callTool( const comparison = await client.callTool(
{ {
name: "compare_functions", name: "compare_functions",
@@ -261,16 +264,15 @@ async function verifyNativeValueAdapters(
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(comparison.isError, true, JSON.stringify(comparison)); assert.notEqual(comparison.isError, true, JSON.stringify(comparison));
const comparisonEvidence = parseEvidence(comparison.structuredContent); assert.notEqual(comparison.structuredContent, undefined);
const comparisonEvidence = parseEvidence(
requireMcpOutput(comparison, "compare_functions"),
);
assert.equal( assert.equal(
comparisonEvidence.normalized_result.status, comparisonEvidence.normalized_result.status,
"unchanged", "unchanged",
JSON.stringify(comparisonEvidence.normalized_result.dimensions), JSON.stringify(comparisonEvidence.normalized_result.dimensions),
); );
assert.deepEqual(
comparisonEvidence.normalized_result,
comparison.structuredContent?.normalized_result,
);
assert.ok( assert.ok(
comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id), comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id),
); );
@@ -346,8 +348,9 @@ async function verifyUnresolvedCallEvidence(client) {
arguments: {}, arguments: {},
}); });
assert.notEqual(inventory.isError, true, JSON.stringify(inventory)); assert.notEqual(inventory.isError, true, JSON.stringify(inventory));
assert.notEqual(inventory.structuredContent, undefined);
const procedures = parseEvidence( const procedures = parseEvidence(
inventory.structuredContent, requireMcpOutput(inventory, "list_procedures"),
).normalized_result; ).normalized_result;
const indirect = procedures.find((item) => const indirect = procedures.find((item) =>
item.value.endsWith("rea_ghidra_inventory_indirect"), item.value.endsWith("rea_ghidra_inventory_indirect"),
@@ -367,7 +370,10 @@ async function verifyUnresolvedCallEvidence(client) {
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed)); assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
const evidence = parseEvidence(analyzed.structuredContent); assert.notEqual(analyzed.structuredContent, undefined);
const evidence = parseEvidence(
requireMcpOutput(analyzed, "analyze_function"),
);
assert.equal(evidence.relationship_coverage?.status, "partial"); assert.equal(evidence.relationship_coverage?.status, "partial");
assert.ok(evidence.normalized_result.unresolved_calls.length > 0); assert.ok(evidence.normalized_result.unresolved_calls.length > 0);
const traced = await client.callTool( const traced = await client.callTool(
@@ -382,7 +388,10 @@ async function verifyUnresolvedCallEvidence(client) {
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(traced.isError, true, JSON.stringify(traced)); assert.notEqual(traced.isError, true, JSON.stringify(traced));
const trace = parseEvidence(traced.structuredContent).normalized_result; assert.notEqual(traced.structuredContent, undefined);
const trace = parseEvidence(
requireMcpOutput(traced, "trace_call_path"),
).normalized_result;
assert.equal(trace.goal_status, "unknown"); assert.equal(trace.goal_status, "unknown");
assert.equal(trace.truncated, true); assert.equal(trace.truncated, true);
assert.ok(trace.residual_unknowns.length > 0); assert.ok(trace.residual_unknowns.length > 0);
@@ -397,8 +406,10 @@ async function verifyUnresolvedCallEvidence(client) {
{ timeout: 180000 }, { timeout: 180000 },
); );
assert.notEqual(compared.isError, true, JSON.stringify(compared)); assert.notEqual(compared.isError, true, JSON.stringify(compared));
assert.notEqual(compared.structuredContent, undefined);
assert.equal( assert.equal(
parseEvidence(compared.structuredContent).normalized_result.status, parseEvidence(requireMcpOutput(compared, "compare_functions"))
.normalized_result.status,
"unknown", "unknown",
); );
} }
+5 -2
View File
@@ -26,7 +26,10 @@ import {
locateApkSigningBlock, locateApkSigningBlock,
parseZipTail, parseZipTail,
} from "../dist/domain/apkSignature/apkSigningBlock.js"; } from "../dist/domain/apkSignature/apkSigningBlock.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; import {
mcpTextValue,
requireMcpOutput,
} from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
const execute = promisify(execFile); const execute = promisify(execFile);
@@ -261,7 +264,7 @@ try {
ajv.validate(advertised.outputSchema, reply.structuredContent), ajv.validate(advertised.outputSchema, reply.structuredContent),
JSON.stringify(ajv.errors), JSON.stringify(ajv.errors),
); );
const mcp = parseEvidence(reply.structuredContent); const mcp = parseEvidence(requireMcpOutput(reply, "inspect_apk_signature"));
assert.deepEqual(cli, mcp, "CLI and MCP evidence differ"); assert.deepEqual(cli, mcp, "CLI and MCP evidence differ");
cases += 2; cases += 2;
return mcp.normalized_result; return mcp.normalized_result;
+13 -22
View File
@@ -1,6 +1,7 @@
import { import {
requireMcpEvidenceBundle, requireMcpEvidenceBundle,
requireMcpLifecycleResult, requireMcpLifecycleResult,
requireMcpOutput,
requireMcpToolError, requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs"; } from "./lib/mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
@@ -172,31 +173,23 @@ const verifyMcpInlineArtifactEvidence = async (
{ name: "inspect_artifact", arguments: {} }, { name: "inspect_artifact", arguments: {} },
mcpOptions, mcpOptions,
); );
assert.notEqual(inspected.isError, true, JSON.stringify(inspected)); assert.notEqual(inspected.structuredContent, undefined);
const source = parseEvidence(inspected.structuredContent); const source = parseEvidence(
assert.deepEqual( requireMcpOutput(inspected, "inspect_artifact"),
source.normalized_result,
inspected.structuredContent?.normalized_result,
);
assert.deepEqual(
JSON.parse(prompts.mcpText(inspected)),
inspected.structuredContent,
); );
const compared = await client.callTool( const compared = await client.callTool(
{ {
name: "compare_artifacts", name: "compare_artifacts",
arguments: { arguments: {
left: inspected.structuredContent, left: source,
right: inspected.structuredContent, right: source,
}, },
}, },
mcpOptions, mcpOptions,
); );
assert.notEqual(compared.isError, true, JSON.stringify(compared)); assert.notEqual(compared.structuredContent, undefined);
const comparison = parseEvidence(compared.structuredContent); const comparison = parseEvidence(
assert.deepEqual( requireMcpOutput(compared, "compare_artifacts"),
comparison.normalized_result,
compared.structuredContent?.normalized_result,
); );
assert.ok(comparison.evidence_links.includes(source.evidence_id)); assert.ok(comparison.evidence_links.includes(source.evidence_id));
const bundleReply = await client.callTool( const bundleReply = await client.callTool(
@@ -220,13 +213,11 @@ const verifyMcpInlineArtifactEvidence = async (
{ name: "extract_artifact", arguments: {} }, { name: "extract_artifact", arguments: {} },
mcpOptions, mcpOptions,
); );
assert.notEqual(extracted.isError, true, JSON.stringify(extracted)); assert.notEqual(extracted.structuredContent, undefined);
const extraction = parseEvidence(extracted.structuredContent); const extraction = parseEvidence(
assert.deepEqual( requireMcpOutput(extracted, "extract_artifact"),
extraction.normalized_result,
extracted.structuredContent?.normalized_result,
); );
const materialized = extracted.structuredContent?.normalized_result; const materialized = extraction.normalized_result;
assert.equal(typeof materialized?.output_root, "string"); assert.equal(typeof materialized?.output_root, "string");
outputRoot = materialized.output_root; outputRoot = materialized.output_root;
assert.equal(materialized.containment_verified, true); assert.equal(materialized.containment_verified, true);
+6 -9
View File
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { tsImport } from "tsx/esm/api"; import { tsImport } from "tsx/esm/api";
import { toolContract } from "../dist/contracts/toolContracts.js"; import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
import { parseEvidence } from "../dist/domain/evidence.js"; import { parseEvidence } from "../dist/domain/evidence.js";
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js"; import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
@@ -80,10 +80,8 @@ await withVerifierCleanup(async () => {
{ name: "binary_session", arguments: {} }, { name: "binary_session", arguments: {} },
{ toolDefinition: sessionTool }, { toolDefinition: sessionTool },
); );
assert.notEqual(sessionReply.isError, true); assert.notEqual(sessionReply.structuredContent, undefined);
const sessionStatus = toolContract("binary_session").outputSchema.parse( const sessionStatus = requireMcpOutput(sessionReply, "binary_session").result;
sessionReply.structuredContent,
).result;
const javascriptMcp = await analyzeMcp(client, javascriptPath); const javascriptMcp = await analyzeMcp(client, javascriptPath);
const electronMcp = await analyzeMcp(client, electronPath); const electronMcp = await analyzeMcp(client, electronPath);
assert.equal( assert.equal(
@@ -194,11 +192,10 @@ async function analyzeMcp(client_, inputPath) {
name: "analyze_javascript_application", name: "analyze_javascript_application",
arguments: { input_path: inputPath }, arguments: { input_path: inputPath },
}); });
assert.notEqual(response.isError, true); assert.notEqual(response.structuredContent, undefined);
toolContract("analyze_javascript_application").outputSchema.parse( return parseEvidence(
response.structuredContent, requireMcpOutput(response, "analyze_javascript_application"),
); );
return parseEvidence(response.structuredContent);
} }
function verifySyntheticEvaluator() { function verifySyntheticEvaluator() {
+3 -2
View File
@@ -21,6 +21,7 @@ import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js"; import { parseEvidence } from "../dist/domain/evidence.js";
import { import {
requireMcpEvidenceBundle, requireMcpEvidenceBundle,
requireMcpOutput,
mcpTextValue, mcpTextValue,
requireMcpToolError, requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs"; } from "./lib/mcp-verifier-results.mjs";
@@ -278,7 +279,7 @@ try {
ajv.validate(advertised.outputSchema, reply.structuredContent), ajv.validate(advertised.outputSchema, reply.structuredContent),
JSON.stringify(ajv.errors), JSON.stringify(ajv.errors),
); );
const mcp = parseEvidence(reply.structuredContent); const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
assert.equal(cli.evidence_id, mcp.evidence_id); assert.equal(cli.evidence_id, mcp.evidence_id);
assert.deepEqual(cli, mcp); assert.deepEqual(cli, mcp);
const report = mcp.normalized_result; const report = mcp.normalized_result;
@@ -516,7 +517,7 @@ try {
ajv.validate(advertised.outputSchema, reply.structuredContent), ajv.validate(advertised.outputSchema, reply.structuredContent),
JSON.stringify(ajv.errors), JSON.stringify(ajv.errors),
); );
const mcp = parseEvidence(reply.structuredContent); const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
assert.deepEqual(cli, mcp); assert.deepEqual(cli, mcp);
const report = mcp.normalized_result; const report = mcp.normalized_result;
assert.equal(report.validation, "valid"); assert.equal(report.validation, "valid");
+5 -2
View File
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client"; import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../dist/domain/evidence.js"; import { parseEvidence } from "../dist/domain/evidence.js";
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
const execute = promisify(execFile); const execute = promisify(execFile);
const bin = process.env.REA_WABT_BIN_DIRECTORY; const bin = process.env.REA_WABT_BIN_DIRECTORY;
@@ -53,7 +54,8 @@ const inspect = async (path) => {
arguments: { path }, arguments: { path },
}); });
assert.notEqual(result.isError, true, JSON.stringify(result)); assert.notEqual(result.isError, true, JSON.stringify(result));
return parseEvidence(result.structuredContent); assert.notEqual(result.structuredContent, undefined);
return parseEvidence(requireMcpOutput(result, "inspect_wasm_artifact"));
}; };
const compare = async (left, right, scope = "both") => { const compare = async (left, right, scope = "both") => {
const input = { const input = {
@@ -66,7 +68,8 @@ const compare = async (left, right, scope = "both") => {
arguments: input, arguments: input,
}); });
assert.notEqual(reply.isError, true, JSON.stringify(reply)); assert.notEqual(reply.isError, true, JSON.stringify(reply));
const mcp = parseEvidence(reply.structuredContent); assert.notEqual(reply.structuredContent, undefined);
const mcp = parseEvidence(requireMcpOutput(reply, "compare_wasm_interfaces"));
const file = join(root, "comparison.json"); const file = join(root, "comparison.json");
await writeFile(file, JSON.stringify({ left, right, scope })); await writeFile(file, JSON.stringify({ left, right, scope }));
const local = parseEvidence( const local = parseEvidence(
+6 -8
View File
@@ -1,3 +1,4 @@
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { execFile } from "node:child_process"; import { execFile } from "node:child_process";
import { access, writeFile } from "node:fs/promises"; import { access, writeFile } from "node:fs/promises";
@@ -147,17 +148,14 @@ try {
{ name: task.name, arguments: task.input }, { name: task.name, arguments: task.input },
{ timeout: 150_000 }, { timeout: 150_000 },
); );
assert.notEqual(response.isError, true, JSON.stringify(response)); assert.notEqual(response.structuredContent, undefined);
const structured = response.structuredContent; const evidence = parseEvidence(requireMcpOutput(response, task.name));
assert.ok(structured !== undefined);
const evidence = parseEvidence(structured);
assert.deepEqual( assert.deepEqual(
evidence.normalized_result, evidence.normalized_result,
cliResults.get(task.name).normalized_result, cliResults.get(task.name).normalized_result,
); );
// Raw observations retain different workspace paths and load durations. // Raw observations retain different workspace paths and load durations.
// Validate each evidence ID independently; compare portable results/intent. // Validate each evidence ID independently; compare portable results/intent.
assert.equal(structured.evidence_id, evidence.evidence_id);
assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters); assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters);
assert.deepEqual(evidence.provider, cliResults.get(task.name).provider); assert.deepEqual(evidence.provider, cliResults.get(task.name).provider);
assert.deepEqual( assert.deepEqual(
@@ -169,7 +167,6 @@ try {
cliResults.get(task.name).limitations, cliResults.get(task.name).limitations,
); );
assert.equal(evidence.confidence, cliResults.get(task.name).confidence); assert.equal(evidence.confidence, cliResults.get(task.name).confidence);
assert.deepEqual(structured.normalized_result, evidence.normalized_result);
console.log(`PASS MCP/CLI parity ${task.name}`); console.log(`PASS MCP/CLI parity ${task.name}`);
} }
const overloadedInput = { const overloadedInput = {
@@ -210,9 +207,10 @@ try {
{ name: "trace_android_references", arguments: overloadedInput }, { name: "trace_android_references", arguments: overloadedInput },
{ timeout: 150_000 }, { timeout: 150_000 },
); );
assert.notEqual(overloadedMcp.isError, true, JSON.stringify(overloadedMcp)); assert.notEqual(overloadedMcp.structuredContent, undefined);
assert.deepEqual( assert.deepEqual(
parseEvidence(overloadedMcp.structuredContent).normalized_result, parseEvidence(requireMcpOutput(overloadedMcp, "trace_android_references"))
.normalized_result,
overloadedEvidence.normalized_result, overloadedEvidence.normalized_result,
); );
console.log("PASS real overloaded method references through CLI/MCP"); console.log("PASS real overloaded method references through CLI/MCP");
+5 -5
View File
@@ -1,6 +1,6 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { parseEvidence } from "../../../dist/domain/evidence.js"; import { parseEvidence } from "../../../dist/domain/evidence.js";
import { requireMcpOperationResult } from "../../lib/mcp-verifier-results.mjs"; import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
/** Compare native selection with an explicit full-inventory oracle on one live session. */ /** Compare native selection with an explicit full-inventory oracle on one live session. */
export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => { export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
@@ -9,11 +9,11 @@ export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
{ name: "search_android_classes", arguments: { path, query } }, { name: "search_android_classes", arguments: { path, query } },
{ timeout: 150_000 }, { timeout: 150_000 },
); );
const result = requireMcpOperationResult( assert.notEqual(response.structuredContent, undefined);
response, const evidence = parseEvidence(
"search_android_classes", requireMcpOutput(response, "search_android_classes"),
); );
const evidence = parseEvidence(response.structuredContent); const result = evidence.normalized_result;
assert.equal(evidence.subject.digest.sha256, sha256); assert.equal(evidence.subject.digest.sha256, sha256);
assert.equal(evidence.parameters.query, query); assert.equal(evidence.parameters.query, query);
assert.equal(result.query, query); assert.equal(result.query, query);
+9 -5
View File
@@ -1,3 +1,4 @@
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { execFile } from "node:child_process"; import { execFile } from "node:child_process";
import { createHash } from "node:crypto"; import { createHash } from "node:crypto";
@@ -172,8 +173,10 @@ try {
{ name: "inspect_artifact", arguments: {} }, { name: "inspect_artifact", arguments: {} },
{ timeout: 150_000 }, { timeout: 150_000 },
); );
assert.notEqual(inspected.isError, true, JSON.stringify(inspected)); assert.notEqual(inspected.structuredContent, undefined);
const mcpInspection = parseEvidence(inspected.structuredContent); const mcpInspection = parseEvidence(
requireMcpOutput(inspected, "inspect_artifact"),
);
assert.deepEqual( assert.deepEqual(
mcpInspection.normalized_result, mcpInspection.normalized_result,
inspection.normalized_result, inspection.normalized_result,
@@ -186,9 +189,10 @@ try {
}, },
{ timeout: 150_000 }, { timeout: 150_000 },
); );
assert.notEqual(response.isError, true, JSON.stringify(response)); assert.notEqual(response.structuredContent, undefined);
assert.ok(response.structuredContent !== undefined); const mcpEvidence = parseEvidence(
const mcpEvidence = parseEvidence(response.structuredContent); requireMcpOutput(response, "project_harmony_application_graph"),
);
assert.deepEqual( assert.deepEqual(
mcpEvidence.normalized_result, mcpEvidence.normalized_result,
projectionEvidence.normalized_result, projectionEvidence.normalized_result,
@@ -1,3 +1,11 @@
import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js";
import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js";
import { digestCanonicalValue } from "../../../src/domain/canonicalDigest.js";
import { createAnalysisExecutionEvidence } from "../../../src/domain/analysisObservation.js";
import {
wasmArtifactFixture,
wasmArtifactExecution,
} from "../../fixtures/wasm/artifact.js";
import { toolContract } from "../../../src/contracts/toolContracts.js"; import { toolContract } from "../../../src/contracts/toolContracts.js";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { readFile } from "node:fs/promises"; import { readFile } from "node:fs/promises";
@@ -423,3 +431,145 @@ it("requires canonical complete tool errors before inspecting their diagnostic f
"Expected an MCP tool failure", "Expected an MCP tool failure",
); );
}); });
it.each(["transport disagreement", "malformed normalized result"])(
"admits complete Wasm Evidence at the interface verifier boundary: %s",
async (scenario) => {
const source = await readFile(
new URL(
"../../../scripts/verify-wasm-interface-comparison.mjs",
import.meta.url,
),
"utf8",
);
const declaration = parse(source, { sourceType: "module" })
.program.body.filter((node) => node.type === "VariableDeclaration")
.flatMap((node) => node.declarations)
.find(
(node) => node.id.type === "Identifier" && node.id.name === "inspect",
)?.init;
if (declaration?.start == null || declaration.end == null)
throw new Error("Wasm interface verifier inspect boundary missing");
const report = wasmArtifactFixture();
const execution = wasmArtifactExecution(report);
const evidence = createAnalysisExecutionEvidence(
"inspect_wasm_artifact",
{ path: report.artifact.path },
execution,
"observed",
);
let response: unknown;
const inspect = runInNewContext(
`(${source.slice(declaration.start, declaration.end)})`,
{
assert,
client: { callTool: async () => response },
parseEvidence,
...(await verifierResults()),
},
) as (path: string) => Promise<unknown>;
response = { ...textReply(evidence), structuredContent: evidence };
await expect(inspect(report.artifact.path)).resolves.toEqual(evidence);
if (scenario === "transport disagreement") {
const different = wasmArtifactFixture();
different.limitations = ["Independent complete Wasm observation."];
const structured = createAnalysisExecutionEvidence(
"inspect_wasm_artifact",
{ path: different.artifact.path },
wasmArtifactExecution(different),
"observed",
);
expect(
parseEvidence(
toolContract("inspect_wasm_artifact").outputSchema.parse(structured),
),
).toEqual(structured);
response = { ...textReply(evidence), structuredContent: structured };
await expect(inspect(report.artifact.path)).rejects.toThrow(
"text and structured results disagree",
);
} else {
const malformed = createEvidence(
execution.subject ?? undefined,
execution.provider,
{
operation: "inspect_wasm_artifact",
parameters: { path: report.artifact.path },
result: "authentic Evidence with an invalid Wasm result shape",
},
);
expect(parseEvidence(malformed)).toEqual(malformed);
response = { ...textReply(malformed), structuredContent: malformed };
await expect(inspect(report.artifact.path)).rejects.toThrow();
}
response = textReply(evidence);
await expect(inspect(report.artifact.path)).rejects.toThrow();
},
);
it("preserves raw extraction manifest extensions for separate Evidence authentication", async () => {
const { requireMcpOutput } = await verifierResults();
const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left;
const inventory = artifactInventoryResultSchema.parse(
inventoryEvidence.normalized_result,
);
const extractionSemantic = {
source_manifest_id: inventory.manifest.manifest_id,
selected_occurrence_ids: [],
files_sha256: digestCanonicalValue([], "Artifact"),
output_root_alias: "$OUTPUT_ROOT",
};
const report = {
manifest: inventory.manifest,
extraction_manifest: {
extraction_id: `aex_${digestCanonicalValue(extractionSemantic, "Artifact")}`,
...extractionSemantic,
},
output_root: "/owned/output",
artifacts: [],
containment_verified: true,
cleanup: { attempted: true, verified: true, residual_paths: [] },
coverage: inventory.coverage,
provenance: inventory.provenance,
integrity_contradictions: inventory.integrity_contradictions,
limitations: [],
};
const evidenceFor = (result: unknown) =>
createEvidence(undefined, inventoryEvidence.provider, {
operation: "extract_artifact",
parameters: {},
result,
});
const evidence = evidenceFor(report);
const extendedResult = {
...report,
manifest: { ...report.manifest, vendor_note: "retained source extension" },
};
const tampered = { ...evidence, normalized_result: extendedResult };
const reply = (value: unknown) => ({
...textReply(value),
structuredContent: value,
});
// Shape projection discards the extension; it cannot authenticate its source ID.
expect(toolContract("extract_artifact").outputSchema.parse(tampered)).toEqual(
evidence,
);
const admittedTampered = requireMcpOutput(
reply(tampered),
"extract_artifact",
);
expect(admittedTampered).toEqual(tampered);
expect(() => parseEvidence(admittedTampered)).toThrow(/semantic identifier/u);
const authentic = evidenceFor(extendedResult);
const projected =
toolContract("extract_artifact").outputSchema.parse(authentic);
expect(projected).toEqual({ ...authentic, normalized_result: report });
const admittedAuthentic = requireMcpOutput(
reply(authentic),
"extract_artifact",
);
expect(admittedAuthentic).toEqual(authentic);
expect(parseEvidence(admittedAuthentic)).toEqual(authentic);
});