mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(verifiers): preserve source Evidence through shared admission
This commit is contained in:
1 parent
f6805e29c3
commit
662ec2e450
14 files changed
+255
-86
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -116,9 +117,12 @@ export async function verifyBrowserNetworkEvidence(
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
const evidence = parseEvidence(result.structuredContent);
|
||||
assert.notEqual(result.structuredContent, undefined);
|
||||
const evidence = parseEvidence(
|
||||
requireMcpOutput(result, "capture_browser_scenario"),
|
||||
);
|
||||
mcpProof = assertBrowserNetworkEvidence(
|
||||
browserScenarioCaptureSchema.parse(evidence.normalized_result),
|
||||
evidence.normalized_result,
|
||||
site.origin,
|
||||
);
|
||||
if (attachment !== undefined) {
|
||||
@@ -131,12 +135,13 @@ export async function verifyBrowserNetworkEvidence(
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
assert.notEqual(connected.isError, true, JSON.stringify(connected));
|
||||
const connectedEvidence = parseEvidence(connected.structuredContent);
|
||||
assert.notEqual(connected.structuredContent, undefined);
|
||||
const connectedEvidence = parseEvidence(
|
||||
requireMcpOutput(connected, "capture_browser_scenario"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
assertBrowserNetworkEvidence(
|
||||
browserScenarioCaptureSchema.parse(
|
||||
connectedEvidence.normalized_result,
|
||||
),
|
||||
connectedEvidence.normalized_result,
|
||||
site.origin,
|
||||
"disconnected-external",
|
||||
),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
@@ -163,11 +164,11 @@ const verifyTransports = async (executable, origin) => {
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
||||
const evidence = parseEvidence(captured.structuredContent);
|
||||
assertNetwork(
|
||||
browserScenarioCaptureSchema.parse(evidence.normalized_result),
|
||||
origin,
|
||||
assert.notEqual(captured.structuredContent, undefined);
|
||||
const evidence = parseEvidence(
|
||||
requireMcpOutput(captured, "capture_browser_scenario"),
|
||||
);
|
||||
assertNetwork(evidence.normalized_result, origin);
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
|
||||
import {
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
@@ -95,10 +98,11 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
||||
const mcpEvidence = parseEvidence(captured.structuredContent);
|
||||
const mcpScreenshot = webScreenshotSchema.parse(
|
||||
mcpEvidence.normalized_result,
|
||||
assert.notEqual(captured.structuredContent, undefined);
|
||||
const mcpEvidence = parseEvidence(
|
||||
requireMcpOutput(captured, "capture_web_screenshot"),
|
||||
);
|
||||
const mcpScreenshot = mcpEvidence.normalized_result;
|
||||
assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact);
|
||||
const compared = comparePngScreenshots({
|
||||
before: screenshot.artifact,
|
||||
|
||||
@@ -46,7 +46,8 @@ export const requireMcpOutput = (
|
||||
if (!isDeepStrictEqual(text, result.structuredContent))
|
||||
throw new Error(`${operation} text and structured results disagree`);
|
||||
}
|
||||
return output;
|
||||
// Downstream parseEvidence must authenticate source facts, not a schema projection.
|
||||
return Object.hasOwn(output, "normalized_result") ? text : output;
|
||||
};
|
||||
|
||||
const operationResultKey = (contract) => {
|
||||
|
||||
@@ -14,6 +14,7 @@ import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrac
|
||||
import {
|
||||
requireMcpEvidenceBundle,
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
} from "./mcp-verifier-results.mjs";
|
||||
import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs";
|
||||
|
||||
@@ -198,12 +199,15 @@ async function verifyNativeValueAdapters(
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
const mcpEvidence = parseEvidence(result.structuredContent);
|
||||
assert.notEqual(result.structuredContent, undefined);
|
||||
const mcpEvidence = parseEvidence(
|
||||
requireMcpOutput(result, "trace_native_values"),
|
||||
);
|
||||
assert.equal(mcpEvidence.operation, "trace_native_values");
|
||||
assert.equal(mcpEvidence.provider.id, "rea-workflow");
|
||||
assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile);
|
||||
const mcpGraph = verifyGraph(
|
||||
result.structuredContent?.normalized_result,
|
||||
mcpEvidence.normalized_result,
|
||||
target,
|
||||
procedure,
|
||||
globalAddress,
|
||||
@@ -218,7 +222,10 @@ async function verifyNativeValueAdapters(
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
||||
const functionEvidence = parseEvidence(analyzed.structuredContent);
|
||||
assert.notEqual(analyzed.structuredContent, undefined);
|
||||
const functionEvidence = parseEvidence(
|
||||
requireMcpOutput(analyzed, "analyze_function"),
|
||||
);
|
||||
assert.equal(functionEvidence.operation, "analyze_function");
|
||||
assert.equal(functionEvidence.provider.id, "ghidra");
|
||||
assert.deepEqual(
|
||||
@@ -246,10 +253,6 @@ async function verifyNativeValueAdapters(
|
||||
functionEvidence.analysis_profile,
|
||||
);
|
||||
assert.deepEqual(functionProfile, upstreamProfile);
|
||||
assert.deepEqual(
|
||||
functionEvidence.normalized_result,
|
||||
analyzed.structuredContent?.normalized_result,
|
||||
);
|
||||
const comparison = await client.callTool(
|
||||
{
|
||||
name: "compare_functions",
|
||||
@@ -261,16 +264,15 @@ async function verifyNativeValueAdapters(
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(comparison.isError, true, JSON.stringify(comparison));
|
||||
const comparisonEvidence = parseEvidence(comparison.structuredContent);
|
||||
assert.notEqual(comparison.structuredContent, undefined);
|
||||
const comparisonEvidence = parseEvidence(
|
||||
requireMcpOutput(comparison, "compare_functions"),
|
||||
);
|
||||
assert.equal(
|
||||
comparisonEvidence.normalized_result.status,
|
||||
"unchanged",
|
||||
JSON.stringify(comparisonEvidence.normalized_result.dimensions),
|
||||
);
|
||||
assert.deepEqual(
|
||||
comparisonEvidence.normalized_result,
|
||||
comparison.structuredContent?.normalized_result,
|
||||
);
|
||||
assert.ok(
|
||||
comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id),
|
||||
);
|
||||
@@ -346,8 +348,9 @@ async function verifyUnresolvedCallEvidence(client) {
|
||||
arguments: {},
|
||||
});
|
||||
assert.notEqual(inventory.isError, true, JSON.stringify(inventory));
|
||||
assert.notEqual(inventory.structuredContent, undefined);
|
||||
const procedures = parseEvidence(
|
||||
inventory.structuredContent,
|
||||
requireMcpOutput(inventory, "list_procedures"),
|
||||
).normalized_result;
|
||||
const indirect = procedures.find((item) =>
|
||||
item.value.endsWith("rea_ghidra_inventory_indirect"),
|
||||
@@ -367,7 +370,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
||||
const evidence = parseEvidence(analyzed.structuredContent);
|
||||
assert.notEqual(analyzed.structuredContent, undefined);
|
||||
const evidence = parseEvidence(
|
||||
requireMcpOutput(analyzed, "analyze_function"),
|
||||
);
|
||||
assert.equal(evidence.relationship_coverage?.status, "partial");
|
||||
assert.ok(evidence.normalized_result.unresolved_calls.length > 0);
|
||||
const traced = await client.callTool(
|
||||
@@ -382,7 +388,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(traced.isError, true, JSON.stringify(traced));
|
||||
const trace = parseEvidence(traced.structuredContent).normalized_result;
|
||||
assert.notEqual(traced.structuredContent, undefined);
|
||||
const trace = parseEvidence(
|
||||
requireMcpOutput(traced, "trace_call_path"),
|
||||
).normalized_result;
|
||||
assert.equal(trace.goal_status, "unknown");
|
||||
assert.equal(trace.truncated, true);
|
||||
assert.ok(trace.residual_unknowns.length > 0);
|
||||
@@ -397,8 +406,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
||||
assert.notEqual(compared.structuredContent, undefined);
|
||||
assert.equal(
|
||||
parseEvidence(compared.structuredContent).normalized_result.status,
|
||||
parseEvidence(requireMcpOutput(compared, "compare_functions"))
|
||||
.normalized_result.status,
|
||||
"unknown",
|
||||
);
|
||||
}
|
||||
@@ -26,7 +26,10 @@ import {
|
||||
locateApkSigningBlock,
|
||||
parseZipTail,
|
||||
} from "../dist/domain/apkSignature/apkSigningBlock.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import {
|
||||
mcpTextValue,
|
||||
requireMcpOutput,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
@@ -261,7 +264,7 @@ try {
|
||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
);
|
||||
const mcp = parseEvidence(reply.structuredContent);
|
||||
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_apk_signature"));
|
||||
assert.deepEqual(cli, mcp, "CLI and MCP evidence differ");
|
||||
cases += 2;
|
||||
return mcp.normalized_result;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
requireMcpEvidenceBundle,
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
@@ -172,31 +173,23 @@ const verifyMcpInlineArtifactEvidence = async (
|
||||
{ name: "inspect_artifact", arguments: {} },
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
|
||||
const source = parseEvidence(inspected.structuredContent);
|
||||
assert.deepEqual(
|
||||
source.normalized_result,
|
||||
inspected.structuredContent?.normalized_result,
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(prompts.mcpText(inspected)),
|
||||
inspected.structuredContent,
|
||||
assert.notEqual(inspected.structuredContent, undefined);
|
||||
const source = parseEvidence(
|
||||
requireMcpOutput(inspected, "inspect_artifact"),
|
||||
);
|
||||
const compared = await client.callTool(
|
||||
{
|
||||
name: "compare_artifacts",
|
||||
arguments: {
|
||||
left: inspected.structuredContent,
|
||||
right: inspected.structuredContent,
|
||||
left: source,
|
||||
right: source,
|
||||
},
|
||||
},
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
||||
const comparison = parseEvidence(compared.structuredContent);
|
||||
assert.deepEqual(
|
||||
comparison.normalized_result,
|
||||
compared.structuredContent?.normalized_result,
|
||||
assert.notEqual(compared.structuredContent, undefined);
|
||||
const comparison = parseEvidence(
|
||||
requireMcpOutput(compared, "compare_artifacts"),
|
||||
);
|
||||
assert.ok(comparison.evidence_links.includes(source.evidence_id));
|
||||
const bundleReply = await client.callTool(
|
||||
@@ -220,13 +213,11 @@ const verifyMcpInlineArtifactEvidence = async (
|
||||
{ name: "extract_artifact", arguments: {} },
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(extracted.isError, true, JSON.stringify(extracted));
|
||||
const extraction = parseEvidence(extracted.structuredContent);
|
||||
assert.deepEqual(
|
||||
extraction.normalized_result,
|
||||
extracted.structuredContent?.normalized_result,
|
||||
assert.notEqual(extracted.structuredContent, undefined);
|
||||
const extraction = parseEvidence(
|
||||
requireMcpOutput(extracted, "extract_artifact"),
|
||||
);
|
||||
const materialized = extracted.structuredContent?.normalized_result;
|
||||
const materialized = extraction.normalized_result;
|
||||
assert.equal(typeof materialized?.output_root, "string");
|
||||
outputRoot = materialized.output_root;
|
||||
assert.equal(materialized.containment_verified, true);
|
||||
|
||||
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { tsImport } from "tsx/esm/api";
|
||||
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
|
||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||
@@ -80,10 +80,8 @@ await withVerifierCleanup(async () => {
|
||||
{ name: "binary_session", arguments: {} },
|
||||
{ toolDefinition: sessionTool },
|
||||
);
|
||||
assert.notEqual(sessionReply.isError, true);
|
||||
const sessionStatus = toolContract("binary_session").outputSchema.parse(
|
||||
sessionReply.structuredContent,
|
||||
).result;
|
||||
assert.notEqual(sessionReply.structuredContent, undefined);
|
||||
const sessionStatus = requireMcpOutput(sessionReply, "binary_session").result;
|
||||
const javascriptMcp = await analyzeMcp(client, javascriptPath);
|
||||
const electronMcp = await analyzeMcp(client, electronPath);
|
||||
assert.equal(
|
||||
@@ -194,11 +192,10 @@ async function analyzeMcp(client_, inputPath) {
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: inputPath },
|
||||
});
|
||||
assert.notEqual(response.isError, true);
|
||||
toolContract("analyze_javascript_application").outputSchema.parse(
|
||||
response.structuredContent,
|
||||
assert.notEqual(response.structuredContent, undefined);
|
||||
return parseEvidence(
|
||||
requireMcpOutput(response, "analyze_javascript_application"),
|
||||
);
|
||||
return parseEvidence(response.structuredContent);
|
||||
}
|
||||
|
||||
function verifySyntheticEvaluator() {
|
||||
|
||||
@@ -21,6 +21,7 @@ import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import {
|
||||
requireMcpEvidenceBundle,
|
||||
requireMcpOutput,
|
||||
mcpTextValue,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
@@ -278,7 +279,7 @@ try {
|
||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
);
|
||||
const mcp = parseEvidence(reply.structuredContent);
|
||||
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
|
||||
assert.equal(cli.evidence_id, mcp.evidence_id);
|
||||
assert.deepEqual(cli, mcp);
|
||||
const report = mcp.normalized_result;
|
||||
@@ -516,7 +517,7 @@ try {
|
||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
);
|
||||
const mcp = parseEvidence(reply.structuredContent);
|
||||
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
|
||||
assert.deepEqual(cli, mcp);
|
||||
const report = mcp.normalized_result;
|
||||
assert.equal(report.validation, "valid");
|
||||
|
||||
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const bin = process.env.REA_WABT_BIN_DIRECTORY;
|
||||
@@ -53,7 +54,8 @@ const inspect = async (path) => {
|
||||
arguments: { path },
|
||||
});
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
return parseEvidence(result.structuredContent);
|
||||
assert.notEqual(result.structuredContent, undefined);
|
||||
return parseEvidence(requireMcpOutput(result, "inspect_wasm_artifact"));
|
||||
};
|
||||
const compare = async (left, right, scope = "both") => {
|
||||
const input = {
|
||||
@@ -66,7 +68,8 @@ const compare = async (left, right, scope = "both") => {
|
||||
arguments: input,
|
||||
});
|
||||
assert.notEqual(reply.isError, true, JSON.stringify(reply));
|
||||
const mcp = parseEvidence(reply.structuredContent);
|
||||
assert.notEqual(reply.structuredContent, undefined);
|
||||
const mcp = parseEvidence(requireMcpOutput(reply, "compare_wasm_interfaces"));
|
||||
const file = join(root, "comparison.json");
|
||||
await writeFile(file, JSON.stringify({ left, right, scope }));
|
||||
const local = parseEvidence(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { access, writeFile } from "node:fs/promises";
|
||||
@@ -147,17 +148,14 @@ try {
|
||||
{ name: task.name, arguments: task.input },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
const structured = response.structuredContent;
|
||||
assert.ok(structured !== undefined);
|
||||
const evidence = parseEvidence(structured);
|
||||
assert.notEqual(response.structuredContent, undefined);
|
||||
const evidence = parseEvidence(requireMcpOutput(response, task.name));
|
||||
assert.deepEqual(
|
||||
evidence.normalized_result,
|
||||
cliResults.get(task.name).normalized_result,
|
||||
);
|
||||
// Raw observations retain different workspace paths and load durations.
|
||||
// Validate each evidence ID independently; compare portable results/intent.
|
||||
assert.equal(structured.evidence_id, evidence.evidence_id);
|
||||
assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters);
|
||||
assert.deepEqual(evidence.provider, cliResults.get(task.name).provider);
|
||||
assert.deepEqual(
|
||||
@@ -169,7 +167,6 @@ try {
|
||||
cliResults.get(task.name).limitations,
|
||||
);
|
||||
assert.equal(evidence.confidence, cliResults.get(task.name).confidence);
|
||||
assert.deepEqual(structured.normalized_result, evidence.normalized_result);
|
||||
console.log(`PASS MCP/CLI parity ${task.name}`);
|
||||
}
|
||||
const overloadedInput = {
|
||||
@@ -210,9 +207,10 @@ try {
|
||||
{ name: "trace_android_references", arguments: overloadedInput },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(overloadedMcp.isError, true, JSON.stringify(overloadedMcp));
|
||||
assert.notEqual(overloadedMcp.structuredContent, undefined);
|
||||
assert.deepEqual(
|
||||
parseEvidence(overloadedMcp.structuredContent).normalized_result,
|
||||
parseEvidence(requireMcpOutput(overloadedMcp, "trace_android_references"))
|
||||
.normalized_result,
|
||||
overloadedEvidence.normalized_result,
|
||||
);
|
||||
console.log("PASS real overloaded method references through CLI/MCP");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { parseEvidence } from "../../../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "../../lib/mcp-verifier-results.mjs";
|
||||
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||
|
||||
/** Compare native selection with an explicit full-inventory oracle on one live session. */
|
||||
export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
|
||||
@@ -9,11 +9,11 @@ export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
|
||||
{ name: "search_android_classes", arguments: { path, query } },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
const result = requireMcpOperationResult(
|
||||
response,
|
||||
"search_android_classes",
|
||||
assert.notEqual(response.structuredContent, undefined);
|
||||
const evidence = parseEvidence(
|
||||
requireMcpOutput(response, "search_android_classes"),
|
||||
);
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
const result = evidence.normalized_result;
|
||||
assert.equal(evidence.subject.digest.sha256, sha256);
|
||||
assert.equal(evidence.parameters.query, query);
|
||||
assert.equal(result.query, query);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -172,8 +173,10 @@ try {
|
||||
{ name: "inspect_artifact", arguments: {} },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
|
||||
const mcpInspection = parseEvidence(inspected.structuredContent);
|
||||
assert.notEqual(inspected.structuredContent, undefined);
|
||||
const mcpInspection = parseEvidence(
|
||||
requireMcpOutput(inspected, "inspect_artifact"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
mcpInspection.normalized_result,
|
||||
inspection.normalized_result,
|
||||
@@ -186,9 +189,10 @@ try {
|
||||
},
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(response.structuredContent !== undefined);
|
||||
const mcpEvidence = parseEvidence(response.structuredContent);
|
||||
assert.notEqual(response.structuredContent, undefined);
|
||||
const mcpEvidence = parseEvidence(
|
||||
requireMcpOutput(response, "project_harmony_application_graph"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
mcpEvidence.normalized_result,
|
||||
projectionEvidence.normalized_result,
|
||||
|
||||
@@ -1,3 +1,11 @@
|
||||
import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js";
|
||||
import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js";
|
||||
import { digestCanonicalValue } from "../../../src/domain/canonicalDigest.js";
|
||||
import { createAnalysisExecutionEvidence } from "../../../src/domain/analysisObservation.js";
|
||||
import {
|
||||
wasmArtifactFixture,
|
||||
wasmArtifactExecution,
|
||||
} from "../../fixtures/wasm/artifact.js";
|
||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
@@ -423,3 +431,145 @@ it("requires canonical complete tool errors before inspecting their diagnostic f
|
||||
"Expected an MCP tool failure",
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["transport disagreement", "malformed normalized result"])(
|
||||
"admits complete Wasm Evidence at the interface verifier boundary: %s",
|
||||
async (scenario) => {
|
||||
const source = await readFile(
|
||||
new URL(
|
||||
"../../../scripts/verify-wasm-interface-comparison.mjs",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
const declaration = parse(source, { sourceType: "module" })
|
||||
.program.body.filter((node) => node.type === "VariableDeclaration")
|
||||
.flatMap((node) => node.declarations)
|
||||
.find(
|
||||
(node) => node.id.type === "Identifier" && node.id.name === "inspect",
|
||||
)?.init;
|
||||
if (declaration?.start == null || declaration.end == null)
|
||||
throw new Error("Wasm interface verifier inspect boundary missing");
|
||||
const report = wasmArtifactFixture();
|
||||
const execution = wasmArtifactExecution(report);
|
||||
const evidence = createAnalysisExecutionEvidence(
|
||||
"inspect_wasm_artifact",
|
||||
{ path: report.artifact.path },
|
||||
execution,
|
||||
"observed",
|
||||
);
|
||||
let response: unknown;
|
||||
const inspect = runInNewContext(
|
||||
`(${source.slice(declaration.start, declaration.end)})`,
|
||||
{
|
||||
assert,
|
||||
client: { callTool: async () => response },
|
||||
parseEvidence,
|
||||
...(await verifierResults()),
|
||||
},
|
||||
) as (path: string) => Promise<unknown>;
|
||||
response = { ...textReply(evidence), structuredContent: evidence };
|
||||
await expect(inspect(report.artifact.path)).resolves.toEqual(evidence);
|
||||
|
||||
if (scenario === "transport disagreement") {
|
||||
const different = wasmArtifactFixture();
|
||||
different.limitations = ["Independent complete Wasm observation."];
|
||||
const structured = createAnalysisExecutionEvidence(
|
||||
"inspect_wasm_artifact",
|
||||
{ path: different.artifact.path },
|
||||
wasmArtifactExecution(different),
|
||||
"observed",
|
||||
);
|
||||
expect(
|
||||
parseEvidence(
|
||||
toolContract("inspect_wasm_artifact").outputSchema.parse(structured),
|
||||
),
|
||||
).toEqual(structured);
|
||||
response = { ...textReply(evidence), structuredContent: structured };
|
||||
await expect(inspect(report.artifact.path)).rejects.toThrow(
|
||||
"text and structured results disagree",
|
||||
);
|
||||
} else {
|
||||
const malformed = createEvidence(
|
||||
execution.subject ?? undefined,
|
||||
execution.provider,
|
||||
{
|
||||
operation: "inspect_wasm_artifact",
|
||||
parameters: { path: report.artifact.path },
|
||||
result: "authentic Evidence with an invalid Wasm result shape",
|
||||
},
|
||||
);
|
||||
expect(parseEvidence(malformed)).toEqual(malformed);
|
||||
response = { ...textReply(malformed), structuredContent: malformed };
|
||||
await expect(inspect(report.artifact.path)).rejects.toThrow();
|
||||
}
|
||||
response = textReply(evidence);
|
||||
await expect(inspect(report.artifact.path)).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves raw extraction manifest extensions for separate Evidence authentication", async () => {
|
||||
const { requireMcpOutput } = await verifierResults();
|
||||
const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left;
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
inventoryEvidence.normalized_result,
|
||||
);
|
||||
const extractionSemantic = {
|
||||
source_manifest_id: inventory.manifest.manifest_id,
|
||||
selected_occurrence_ids: [],
|
||||
files_sha256: digestCanonicalValue([], "Artifact"),
|
||||
output_root_alias: "$OUTPUT_ROOT",
|
||||
};
|
||||
const report = {
|
||||
manifest: inventory.manifest,
|
||||
extraction_manifest: {
|
||||
extraction_id: `aex_${digestCanonicalValue(extractionSemantic, "Artifact")}`,
|
||||
...extractionSemantic,
|
||||
},
|
||||
output_root: "/owned/output",
|
||||
artifacts: [],
|
||||
containment_verified: true,
|
||||
cleanup: { attempted: true, verified: true, residual_paths: [] },
|
||||
coverage: inventory.coverage,
|
||||
provenance: inventory.provenance,
|
||||
integrity_contradictions: inventory.integrity_contradictions,
|
||||
limitations: [],
|
||||
};
|
||||
const evidenceFor = (result: unknown) =>
|
||||
createEvidence(undefined, inventoryEvidence.provider, {
|
||||
operation: "extract_artifact",
|
||||
parameters: {},
|
||||
result,
|
||||
});
|
||||
const evidence = evidenceFor(report);
|
||||
const extendedResult = {
|
||||
...report,
|
||||
manifest: { ...report.manifest, vendor_note: "retained source extension" },
|
||||
};
|
||||
const tampered = { ...evidence, normalized_result: extendedResult };
|
||||
const reply = (value: unknown) => ({
|
||||
...textReply(value),
|
||||
structuredContent: value,
|
||||
});
|
||||
// Shape projection discards the extension; it cannot authenticate its source ID.
|
||||
expect(toolContract("extract_artifact").outputSchema.parse(tampered)).toEqual(
|
||||
evidence,
|
||||
);
|
||||
const admittedTampered = requireMcpOutput(
|
||||
reply(tampered),
|
||||
"extract_artifact",
|
||||
);
|
||||
expect(admittedTampered).toEqual(tampered);
|
||||
expect(() => parseEvidence(admittedTampered)).toThrow(/semantic identifier/u);
|
||||
|
||||
const authentic = evidenceFor(extendedResult);
|
||||
const projected =
|
||||
toolContract("extract_artifact").outputSchema.parse(authentic);
|
||||
expect(projected).toEqual({ ...authentic, normalized_result: report });
|
||||
const admittedAuthentic = requireMcpOutput(
|
||||
reply(authentic),
|
||||
"extract_artifact",
|
||||
);
|
||||
expect(admittedAuthentic).toEqual(authentic);
|
||||
expect(parseEvidence(admittedAuthentic)).toEqual(authentic);
|
||||
});
|
||||
Reference in new issue
Block a user