mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(verifiers): preserve source Evidence through shared admission
This commit is contained in:
1 parent
f6805e29c3
commit
662ec2e450
14 files changed
+254
-85
No files matched your search
@@ -1,3 +1,4 @@
|
|||||||
|
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
@@ -116,9 +117,12 @@ export async function verifyBrowserNetworkEvidence(
|
|||||||
{ timeout: 60_000 },
|
{ timeout: 60_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||||
const evidence = parseEvidence(result.structuredContent);
|
assert.notEqual(result.structuredContent, undefined);
|
||||||
|
const evidence = parseEvidence(
|
||||||
|
requireMcpOutput(result, "capture_browser_scenario"),
|
||||||
|
);
|
||||||
mcpProof = assertBrowserNetworkEvidence(
|
mcpProof = assertBrowserNetworkEvidence(
|
||||||
browserScenarioCaptureSchema.parse(evidence.normalized_result),
|
evidence.normalized_result,
|
||||||
site.origin,
|
site.origin,
|
||||||
);
|
);
|
||||||
if (attachment !== undefined) {
|
if (attachment !== undefined) {
|
||||||
@@ -131,12 +135,13 @@ export async function verifyBrowserNetworkEvidence(
|
|||||||
{ timeout: 60_000 },
|
{ timeout: 60_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(connected.isError, true, JSON.stringify(connected));
|
assert.notEqual(connected.isError, true, JSON.stringify(connected));
|
||||||
const connectedEvidence = parseEvidence(connected.structuredContent);
|
assert.notEqual(connected.structuredContent, undefined);
|
||||||
|
const connectedEvidence = parseEvidence(
|
||||||
|
requireMcpOutput(connected, "capture_browser_scenario"),
|
||||||
|
);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
assertBrowserNetworkEvidence(
|
assertBrowserNetworkEvidence(
|
||||||
browserScenarioCaptureSchema.parse(
|
|
||||||
connectedEvidence.normalized_result,
|
connectedEvidence.normalized_result,
|
||||||
),
|
|
||||||
site.origin,
|
site.origin,
|
||||||
"disconnected-external",
|
"disconnected-external",
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { requireMcpOutput } from "./mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import { once } from "node:events";
|
import { once } from "node:events";
|
||||||
@@ -163,11 +164,11 @@ const verifyTransports = async (executable, origin) => {
|
|||||||
{ timeout: 60_000 },
|
{ timeout: 60_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
||||||
const evidence = parseEvidence(captured.structuredContent);
|
assert.notEqual(captured.structuredContent, undefined);
|
||||||
assertNetwork(
|
const evidence = parseEvidence(
|
||||||
browserScenarioCaptureSchema.parse(evidence.normalized_result),
|
requireMcpOutput(captured, "capture_browser_scenario"),
|
||||||
origin,
|
|
||||||
);
|
);
|
||||||
|
assertNetwork(evidence.normalized_result, origin);
|
||||||
} finally {
|
} finally {
|
||||||
try {
|
try {
|
||||||
await client.close();
|
await client.close();
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
|
import {
|
||||||
|
requireMcpOutput,
|
||||||
|
requireMcpToolError,
|
||||||
|
} from "./mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
@@ -95,10 +98,11 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
|
|||||||
{ timeout: 60000 },
|
{ timeout: 60000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
assert.notEqual(captured.isError, true, JSON.stringify(captured));
|
||||||
const mcpEvidence = parseEvidence(captured.structuredContent);
|
assert.notEqual(captured.structuredContent, undefined);
|
||||||
const mcpScreenshot = webScreenshotSchema.parse(
|
const mcpEvidence = parseEvidence(
|
||||||
mcpEvidence.normalized_result,
|
requireMcpOutput(captured, "capture_web_screenshot"),
|
||||||
);
|
);
|
||||||
|
const mcpScreenshot = mcpEvidence.normalized_result;
|
||||||
assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact);
|
assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact);
|
||||||
const compared = comparePngScreenshots({
|
const compared = comparePngScreenshots({
|
||||||
before: screenshot.artifact,
|
before: screenshot.artifact,
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ export const requireMcpOutput = (
|
|||||||
if (!isDeepStrictEqual(text, result.structuredContent))
|
if (!isDeepStrictEqual(text, result.structuredContent))
|
||||||
throw new Error(`${operation} text and structured results disagree`);
|
throw new Error(`${operation} text and structured results disagree`);
|
||||||
}
|
}
|
||||||
return output;
|
// Downstream parseEvidence must authenticate source facts, not a schema projection.
|
||||||
|
return Object.hasOwn(output, "normalized_result") ? text : output;
|
||||||
};
|
};
|
||||||
|
|
||||||
const operationResultKey = (contract) => {
|
const operationResultKey = (contract) => {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrac
|
|||||||
import {
|
import {
|
||||||
requireMcpEvidenceBundle,
|
requireMcpEvidenceBundle,
|
||||||
requireMcpLifecycleResult,
|
requireMcpLifecycleResult,
|
||||||
|
requireMcpOutput,
|
||||||
} from "./mcp-verifier-results.mjs";
|
} from "./mcp-verifier-results.mjs";
|
||||||
import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs";
|
import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs";
|
||||||
|
|
||||||
@@ -198,12 +199,15 @@ async function verifyNativeValueAdapters(
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||||
const mcpEvidence = parseEvidence(result.structuredContent);
|
assert.notEqual(result.structuredContent, undefined);
|
||||||
|
const mcpEvidence = parseEvidence(
|
||||||
|
requireMcpOutput(result, "trace_native_values"),
|
||||||
|
);
|
||||||
assert.equal(mcpEvidence.operation, "trace_native_values");
|
assert.equal(mcpEvidence.operation, "trace_native_values");
|
||||||
assert.equal(mcpEvidence.provider.id, "rea-workflow");
|
assert.equal(mcpEvidence.provider.id, "rea-workflow");
|
||||||
assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile);
|
assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile);
|
||||||
const mcpGraph = verifyGraph(
|
const mcpGraph = verifyGraph(
|
||||||
result.structuredContent?.normalized_result,
|
mcpEvidence.normalized_result,
|
||||||
target,
|
target,
|
||||||
procedure,
|
procedure,
|
||||||
globalAddress,
|
globalAddress,
|
||||||
@@ -218,7 +222,10 @@ async function verifyNativeValueAdapters(
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
||||||
const functionEvidence = parseEvidence(analyzed.structuredContent);
|
assert.notEqual(analyzed.structuredContent, undefined);
|
||||||
|
const functionEvidence = parseEvidence(
|
||||||
|
requireMcpOutput(analyzed, "analyze_function"),
|
||||||
|
);
|
||||||
assert.equal(functionEvidence.operation, "analyze_function");
|
assert.equal(functionEvidence.operation, "analyze_function");
|
||||||
assert.equal(functionEvidence.provider.id, "ghidra");
|
assert.equal(functionEvidence.provider.id, "ghidra");
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
@@ -246,10 +253,6 @@ async function verifyNativeValueAdapters(
|
|||||||
functionEvidence.analysis_profile,
|
functionEvidence.analysis_profile,
|
||||||
);
|
);
|
||||||
assert.deepEqual(functionProfile, upstreamProfile);
|
assert.deepEqual(functionProfile, upstreamProfile);
|
||||||
assert.deepEqual(
|
|
||||||
functionEvidence.normalized_result,
|
|
||||||
analyzed.structuredContent?.normalized_result,
|
|
||||||
);
|
|
||||||
const comparison = await client.callTool(
|
const comparison = await client.callTool(
|
||||||
{
|
{
|
||||||
name: "compare_functions",
|
name: "compare_functions",
|
||||||
@@ -261,16 +264,15 @@ async function verifyNativeValueAdapters(
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(comparison.isError, true, JSON.stringify(comparison));
|
assert.notEqual(comparison.isError, true, JSON.stringify(comparison));
|
||||||
const comparisonEvidence = parseEvidence(comparison.structuredContent);
|
assert.notEqual(comparison.structuredContent, undefined);
|
||||||
|
const comparisonEvidence = parseEvidence(
|
||||||
|
requireMcpOutput(comparison, "compare_functions"),
|
||||||
|
);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
comparisonEvidence.normalized_result.status,
|
comparisonEvidence.normalized_result.status,
|
||||||
"unchanged",
|
"unchanged",
|
||||||
JSON.stringify(comparisonEvidence.normalized_result.dimensions),
|
JSON.stringify(comparisonEvidence.normalized_result.dimensions),
|
||||||
);
|
);
|
||||||
assert.deepEqual(
|
|
||||||
comparisonEvidence.normalized_result,
|
|
||||||
comparison.structuredContent?.normalized_result,
|
|
||||||
);
|
|
||||||
assert.ok(
|
assert.ok(
|
||||||
comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id),
|
comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id),
|
||||||
);
|
);
|
||||||
@@ -346,8 +348,9 @@ async function verifyUnresolvedCallEvidence(client) {
|
|||||||
arguments: {},
|
arguments: {},
|
||||||
});
|
});
|
||||||
assert.notEqual(inventory.isError, true, JSON.stringify(inventory));
|
assert.notEqual(inventory.isError, true, JSON.stringify(inventory));
|
||||||
|
assert.notEqual(inventory.structuredContent, undefined);
|
||||||
const procedures = parseEvidence(
|
const procedures = parseEvidence(
|
||||||
inventory.structuredContent,
|
requireMcpOutput(inventory, "list_procedures"),
|
||||||
).normalized_result;
|
).normalized_result;
|
||||||
const indirect = procedures.find((item) =>
|
const indirect = procedures.find((item) =>
|
||||||
item.value.endsWith("rea_ghidra_inventory_indirect"),
|
item.value.endsWith("rea_ghidra_inventory_indirect"),
|
||||||
@@ -367,7 +370,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
||||||
const evidence = parseEvidence(analyzed.structuredContent);
|
assert.notEqual(analyzed.structuredContent, undefined);
|
||||||
|
const evidence = parseEvidence(
|
||||||
|
requireMcpOutput(analyzed, "analyze_function"),
|
||||||
|
);
|
||||||
assert.equal(evidence.relationship_coverage?.status, "partial");
|
assert.equal(evidence.relationship_coverage?.status, "partial");
|
||||||
assert.ok(evidence.normalized_result.unresolved_calls.length > 0);
|
assert.ok(evidence.normalized_result.unresolved_calls.length > 0);
|
||||||
const traced = await client.callTool(
|
const traced = await client.callTool(
|
||||||
@@ -382,7 +388,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(traced.isError, true, JSON.stringify(traced));
|
assert.notEqual(traced.isError, true, JSON.stringify(traced));
|
||||||
const trace = parseEvidence(traced.structuredContent).normalized_result;
|
assert.notEqual(traced.structuredContent, undefined);
|
||||||
|
const trace = parseEvidence(
|
||||||
|
requireMcpOutput(traced, "trace_call_path"),
|
||||||
|
).normalized_result;
|
||||||
assert.equal(trace.goal_status, "unknown");
|
assert.equal(trace.goal_status, "unknown");
|
||||||
assert.equal(trace.truncated, true);
|
assert.equal(trace.truncated, true);
|
||||||
assert.ok(trace.residual_unknowns.length > 0);
|
assert.ok(trace.residual_unknowns.length > 0);
|
||||||
@@ -397,8 +406,10 @@ async function verifyUnresolvedCallEvidence(client) {
|
|||||||
{ timeout: 180000 },
|
{ timeout: 180000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
||||||
|
assert.notEqual(compared.structuredContent, undefined);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
parseEvidence(compared.structuredContent).normalized_result.status,
|
parseEvidence(requireMcpOutput(compared, "compare_functions"))
|
||||||
|
.normalized_result.status,
|
||||||
"unknown",
|
"unknown",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,10 @@ import {
|
|||||||
locateApkSigningBlock,
|
locateApkSigningBlock,
|
||||||
parseZipTail,
|
parseZipTail,
|
||||||
} from "../dist/domain/apkSignature/apkSigningBlock.js";
|
} from "../dist/domain/apkSignature/apkSigningBlock.js";
|
||||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
import {
|
||||||
|
mcpTextValue,
|
||||||
|
requireMcpOutput,
|
||||||
|
} from "./lib/mcp-verifier-results.mjs";
|
||||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||||
|
|
||||||
const execute = promisify(execFile);
|
const execute = promisify(execFile);
|
||||||
@@ -261,7 +264,7 @@ try {
|
|||||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||||
JSON.stringify(ajv.errors),
|
JSON.stringify(ajv.errors),
|
||||||
);
|
);
|
||||||
const mcp = parseEvidence(reply.structuredContent);
|
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_apk_signature"));
|
||||||
assert.deepEqual(cli, mcp, "CLI and MCP evidence differ");
|
assert.deepEqual(cli, mcp, "CLI and MCP evidence differ");
|
||||||
cases += 2;
|
cases += 2;
|
||||||
return mcp.normalized_result;
|
return mcp.normalized_result;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import {
|
import {
|
||||||
requireMcpEvidenceBundle,
|
requireMcpEvidenceBundle,
|
||||||
requireMcpLifecycleResult,
|
requireMcpLifecycleResult,
|
||||||
|
requireMcpOutput,
|
||||||
requireMcpToolError,
|
requireMcpToolError,
|
||||||
} from "./lib/mcp-verifier-results.mjs";
|
} from "./lib/mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
@@ -172,31 +173,23 @@ const verifyMcpInlineArtifactEvidence = async (
|
|||||||
{ name: "inspect_artifact", arguments: {} },
|
{ name: "inspect_artifact", arguments: {} },
|
||||||
mcpOptions,
|
mcpOptions,
|
||||||
);
|
);
|
||||||
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
|
assert.notEqual(inspected.structuredContent, undefined);
|
||||||
const source = parseEvidence(inspected.structuredContent);
|
const source = parseEvidence(
|
||||||
assert.deepEqual(
|
requireMcpOutput(inspected, "inspect_artifact"),
|
||||||
source.normalized_result,
|
|
||||||
inspected.structuredContent?.normalized_result,
|
|
||||||
);
|
|
||||||
assert.deepEqual(
|
|
||||||
JSON.parse(prompts.mcpText(inspected)),
|
|
||||||
inspected.structuredContent,
|
|
||||||
);
|
);
|
||||||
const compared = await client.callTool(
|
const compared = await client.callTool(
|
||||||
{
|
{
|
||||||
name: "compare_artifacts",
|
name: "compare_artifacts",
|
||||||
arguments: {
|
arguments: {
|
||||||
left: inspected.structuredContent,
|
left: source,
|
||||||
right: inspected.structuredContent,
|
right: source,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
mcpOptions,
|
mcpOptions,
|
||||||
);
|
);
|
||||||
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
assert.notEqual(compared.structuredContent, undefined);
|
||||||
const comparison = parseEvidence(compared.structuredContent);
|
const comparison = parseEvidence(
|
||||||
assert.deepEqual(
|
requireMcpOutput(compared, "compare_artifacts"),
|
||||||
comparison.normalized_result,
|
|
||||||
compared.structuredContent?.normalized_result,
|
|
||||||
);
|
);
|
||||||
assert.ok(comparison.evidence_links.includes(source.evidence_id));
|
assert.ok(comparison.evidence_links.includes(source.evidence_id));
|
||||||
const bundleReply = await client.callTool(
|
const bundleReply = await client.callTool(
|
||||||
@@ -220,13 +213,11 @@ const verifyMcpInlineArtifactEvidence = async (
|
|||||||
{ name: "extract_artifact", arguments: {} },
|
{ name: "extract_artifact", arguments: {} },
|
||||||
mcpOptions,
|
mcpOptions,
|
||||||
);
|
);
|
||||||
assert.notEqual(extracted.isError, true, JSON.stringify(extracted));
|
assert.notEqual(extracted.structuredContent, undefined);
|
||||||
const extraction = parseEvidence(extracted.structuredContent);
|
const extraction = parseEvidence(
|
||||||
assert.deepEqual(
|
requireMcpOutput(extracted, "extract_artifact"),
|
||||||
extraction.normalized_result,
|
|
||||||
extracted.structuredContent?.normalized_result,
|
|
||||||
);
|
);
|
||||||
const materialized = extracted.structuredContent?.normalized_result;
|
const materialized = extraction.normalized_result;
|
||||||
assert.equal(typeof materialized?.output_root, "string");
|
assert.equal(typeof materialized?.output_root, "string");
|
||||||
outputRoot = materialized.output_root;
|
outputRoot = materialized.output_root;
|
||||||
assert.equal(materialized.containment_verified, true);
|
assert.equal(materialized.containment_verified, true);
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client";
|
|||||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||||
import { tsImport } from "tsx/esm/api";
|
import { tsImport } from "tsx/esm/api";
|
||||||
|
|
||||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||||
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
|
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
|
||||||
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
||||||
@@ -80,10 +80,8 @@ await withVerifierCleanup(async () => {
|
|||||||
{ name: "binary_session", arguments: {} },
|
{ name: "binary_session", arguments: {} },
|
||||||
{ toolDefinition: sessionTool },
|
{ toolDefinition: sessionTool },
|
||||||
);
|
);
|
||||||
assert.notEqual(sessionReply.isError, true);
|
assert.notEqual(sessionReply.structuredContent, undefined);
|
||||||
const sessionStatus = toolContract("binary_session").outputSchema.parse(
|
const sessionStatus = requireMcpOutput(sessionReply, "binary_session").result;
|
||||||
sessionReply.structuredContent,
|
|
||||||
).result;
|
|
||||||
const javascriptMcp = await analyzeMcp(client, javascriptPath);
|
const javascriptMcp = await analyzeMcp(client, javascriptPath);
|
||||||
const electronMcp = await analyzeMcp(client, electronPath);
|
const electronMcp = await analyzeMcp(client, electronPath);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
@@ -194,11 +192,10 @@ async function analyzeMcp(client_, inputPath) {
|
|||||||
name: "analyze_javascript_application",
|
name: "analyze_javascript_application",
|
||||||
arguments: { input_path: inputPath },
|
arguments: { input_path: inputPath },
|
||||||
});
|
});
|
||||||
assert.notEqual(response.isError, true);
|
assert.notEqual(response.structuredContent, undefined);
|
||||||
toolContract("analyze_javascript_application").outputSchema.parse(
|
return parseEvidence(
|
||||||
response.structuredContent,
|
requireMcpOutput(response, "analyze_javascript_application"),
|
||||||
);
|
);
|
||||||
return parseEvidence(response.structuredContent);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function verifySyntheticEvaluator() {
|
function verifySyntheticEvaluator() {
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
|||||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||||
import {
|
import {
|
||||||
requireMcpEvidenceBundle,
|
requireMcpEvidenceBundle,
|
||||||
|
requireMcpOutput,
|
||||||
mcpTextValue,
|
mcpTextValue,
|
||||||
requireMcpToolError,
|
requireMcpToolError,
|
||||||
} from "./lib/mcp-verifier-results.mjs";
|
} from "./lib/mcp-verifier-results.mjs";
|
||||||
@@ -278,7 +279,7 @@ try {
|
|||||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||||
JSON.stringify(ajv.errors),
|
JSON.stringify(ajv.errors),
|
||||||
);
|
);
|
||||||
const mcp = parseEvidence(reply.structuredContent);
|
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
|
||||||
assert.equal(cli.evidence_id, mcp.evidence_id);
|
assert.equal(cli.evidence_id, mcp.evidence_id);
|
||||||
assert.deepEqual(cli, mcp);
|
assert.deepEqual(cli, mcp);
|
||||||
const report = mcp.normalized_result;
|
const report = mcp.normalized_result;
|
||||||
@@ -516,7 +517,7 @@ try {
|
|||||||
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
ajv.validate(advertised.outputSchema, reply.structuredContent),
|
||||||
JSON.stringify(ajv.errors),
|
JSON.stringify(ajv.errors),
|
||||||
);
|
);
|
||||||
const mcp = parseEvidence(reply.structuredContent);
|
const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact"));
|
||||||
assert.deepEqual(cli, mcp);
|
assert.deepEqual(cli, mcp);
|
||||||
const report = mcp.normalized_result;
|
const report = mcp.normalized_result;
|
||||||
assert.equal(report.validation, "valid");
|
assert.equal(report.validation, "valid");
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { promisify } from "node:util";
|
|||||||
import { Client } from "@modelcontextprotocol/client";
|
import { Client } from "@modelcontextprotocol/client";
|
||||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||||
|
import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
|
||||||
|
|
||||||
const execute = promisify(execFile);
|
const execute = promisify(execFile);
|
||||||
const bin = process.env.REA_WABT_BIN_DIRECTORY;
|
const bin = process.env.REA_WABT_BIN_DIRECTORY;
|
||||||
@@ -53,7 +54,8 @@ const inspect = async (path) => {
|
|||||||
arguments: { path },
|
arguments: { path },
|
||||||
});
|
});
|
||||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||||
return parseEvidence(result.structuredContent);
|
assert.notEqual(result.structuredContent, undefined);
|
||||||
|
return parseEvidence(requireMcpOutput(result, "inspect_wasm_artifact"));
|
||||||
};
|
};
|
||||||
const compare = async (left, right, scope = "both") => {
|
const compare = async (left, right, scope = "both") => {
|
||||||
const input = {
|
const input = {
|
||||||
@@ -66,7 +68,8 @@ const compare = async (left, right, scope = "both") => {
|
|||||||
arguments: input,
|
arguments: input,
|
||||||
});
|
});
|
||||||
assert.notEqual(reply.isError, true, JSON.stringify(reply));
|
assert.notEqual(reply.isError, true, JSON.stringify(reply));
|
||||||
const mcp = parseEvidence(reply.structuredContent);
|
assert.notEqual(reply.structuredContent, undefined);
|
||||||
|
const mcp = parseEvidence(requireMcpOutput(reply, "compare_wasm_interfaces"));
|
||||||
const file = join(root, "comparison.json");
|
const file = join(root, "comparison.json");
|
||||||
await writeFile(file, JSON.stringify({ left, right, scope }));
|
await writeFile(file, JSON.stringify({ left, right, scope }));
|
||||||
const local = parseEvidence(
|
const local = parseEvidence(
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import { access, writeFile } from "node:fs/promises";
|
import { access, writeFile } from "node:fs/promises";
|
||||||
@@ -147,17 +148,14 @@ try {
|
|||||||
{ name: task.name, arguments: task.input },
|
{ name: task.name, arguments: task.input },
|
||||||
{ timeout: 150_000 },
|
{ timeout: 150_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
assert.notEqual(response.structuredContent, undefined);
|
||||||
const structured = response.structuredContent;
|
const evidence = parseEvidence(requireMcpOutput(response, task.name));
|
||||||
assert.ok(structured !== undefined);
|
|
||||||
const evidence = parseEvidence(structured);
|
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
evidence.normalized_result,
|
evidence.normalized_result,
|
||||||
cliResults.get(task.name).normalized_result,
|
cliResults.get(task.name).normalized_result,
|
||||||
);
|
);
|
||||||
// Raw observations retain different workspace paths and load durations.
|
// Raw observations retain different workspace paths and load durations.
|
||||||
// Validate each evidence ID independently; compare portable results/intent.
|
// Validate each evidence ID independently; compare portable results/intent.
|
||||||
assert.equal(structured.evidence_id, evidence.evidence_id);
|
|
||||||
assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters);
|
assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters);
|
||||||
assert.deepEqual(evidence.provider, cliResults.get(task.name).provider);
|
assert.deepEqual(evidence.provider, cliResults.get(task.name).provider);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
@@ -169,7 +167,6 @@ try {
|
|||||||
cliResults.get(task.name).limitations,
|
cliResults.get(task.name).limitations,
|
||||||
);
|
);
|
||||||
assert.equal(evidence.confidence, cliResults.get(task.name).confidence);
|
assert.equal(evidence.confidence, cliResults.get(task.name).confidence);
|
||||||
assert.deepEqual(structured.normalized_result, evidence.normalized_result);
|
|
||||||
console.log(`PASS MCP/CLI parity ${task.name}`);
|
console.log(`PASS MCP/CLI parity ${task.name}`);
|
||||||
}
|
}
|
||||||
const overloadedInput = {
|
const overloadedInput = {
|
||||||
@@ -210,9 +207,10 @@ try {
|
|||||||
{ name: "trace_android_references", arguments: overloadedInput },
|
{ name: "trace_android_references", arguments: overloadedInput },
|
||||||
{ timeout: 150_000 },
|
{ timeout: 150_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(overloadedMcp.isError, true, JSON.stringify(overloadedMcp));
|
assert.notEqual(overloadedMcp.structuredContent, undefined);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
parseEvidence(overloadedMcp.structuredContent).normalized_result,
|
parseEvidence(requireMcpOutput(overloadedMcp, "trace_android_references"))
|
||||||
|
.normalized_result,
|
||||||
overloadedEvidence.normalized_result,
|
overloadedEvidence.normalized_result,
|
||||||
);
|
);
|
||||||
console.log("PASS real overloaded method references through CLI/MCP");
|
console.log("PASS real overloaded method references through CLI/MCP");
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { parseEvidence } from "../../../dist/domain/evidence.js";
|
import { parseEvidence } from "../../../dist/domain/evidence.js";
|
||||||
import { requireMcpOperationResult } from "../../lib/mcp-verifier-results.mjs";
|
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||||
|
|
||||||
/** Compare native selection with an explicit full-inventory oracle on one live session. */
|
/** Compare native selection with an explicit full-inventory oracle on one live session. */
|
||||||
export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
|
export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
|
||||||
@@ -9,11 +9,11 @@ export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => {
|
|||||||
{ name: "search_android_classes", arguments: { path, query } },
|
{ name: "search_android_classes", arguments: { path, query } },
|
||||||
{ timeout: 150_000 },
|
{ timeout: 150_000 },
|
||||||
);
|
);
|
||||||
const result = requireMcpOperationResult(
|
assert.notEqual(response.structuredContent, undefined);
|
||||||
response,
|
const evidence = parseEvidence(
|
||||||
"search_android_classes",
|
requireMcpOutput(response, "search_android_classes"),
|
||||||
);
|
);
|
||||||
const evidence = parseEvidence(response.structuredContent);
|
const result = evidence.normalized_result;
|
||||||
assert.equal(evidence.subject.digest.sha256, sha256);
|
assert.equal(evidence.subject.digest.sha256, sha256);
|
||||||
assert.equal(evidence.parameters.query, query);
|
assert.equal(evidence.parameters.query, query);
|
||||||
assert.equal(result.query, query);
|
assert.equal(result.query, query);
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { execFile } from "node:child_process";
|
import { execFile } from "node:child_process";
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
@@ -172,8 +173,10 @@ try {
|
|||||||
{ name: "inspect_artifact", arguments: {} },
|
{ name: "inspect_artifact", arguments: {} },
|
||||||
{ timeout: 150_000 },
|
{ timeout: 150_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
|
assert.notEqual(inspected.structuredContent, undefined);
|
||||||
const mcpInspection = parseEvidence(inspected.structuredContent);
|
const mcpInspection = parseEvidence(
|
||||||
|
requireMcpOutput(inspected, "inspect_artifact"),
|
||||||
|
);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
mcpInspection.normalized_result,
|
mcpInspection.normalized_result,
|
||||||
inspection.normalized_result,
|
inspection.normalized_result,
|
||||||
@@ -186,9 +189,10 @@ try {
|
|||||||
},
|
},
|
||||||
{ timeout: 150_000 },
|
{ timeout: 150_000 },
|
||||||
);
|
);
|
||||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
assert.notEqual(response.structuredContent, undefined);
|
||||||
assert.ok(response.structuredContent !== undefined);
|
const mcpEvidence = parseEvidence(
|
||||||
const mcpEvidence = parseEvidence(response.structuredContent);
|
requireMcpOutput(response, "project_harmony_application_graph"),
|
||||||
|
);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
mcpEvidence.normalized_result,
|
mcpEvidence.normalized_result,
|
||||||
projectionEvidence.normalized_result,
|
projectionEvidence.normalized_result,
|
||||||
|
|||||||
@@ -1,3 +1,11 @@
|
|||||||
|
import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js";
|
||||||
|
import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js";
|
||||||
|
import { digestCanonicalValue } from "../../../src/domain/canonicalDigest.js";
|
||||||
|
import { createAnalysisExecutionEvidence } from "../../../src/domain/analysisObservation.js";
|
||||||
|
import {
|
||||||
|
wasmArtifactFixture,
|
||||||
|
wasmArtifactExecution,
|
||||||
|
} from "../../fixtures/wasm/artifact.js";
|
||||||
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
import { toolContract } from "../../../src/contracts/toolContracts.js";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { readFile } from "node:fs/promises";
|
import { readFile } from "node:fs/promises";
|
||||||
@@ -423,3 +431,145 @@ it("requires canonical complete tool errors before inspecting their diagnostic f
|
|||||||
"Expected an MCP tool failure",
|
"Expected an MCP tool failure",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each(["transport disagreement", "malformed normalized result"])(
|
||||||
|
"admits complete Wasm Evidence at the interface verifier boundary: %s",
|
||||||
|
async (scenario) => {
|
||||||
|
const source = await readFile(
|
||||||
|
new URL(
|
||||||
|
"../../../scripts/verify-wasm-interface-comparison.mjs",
|
||||||
|
import.meta.url,
|
||||||
|
),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const declaration = parse(source, { sourceType: "module" })
|
||||||
|
.program.body.filter((node) => node.type === "VariableDeclaration")
|
||||||
|
.flatMap((node) => node.declarations)
|
||||||
|
.find(
|
||||||
|
(node) => node.id.type === "Identifier" && node.id.name === "inspect",
|
||||||
|
)?.init;
|
||||||
|
if (declaration?.start == null || declaration.end == null)
|
||||||
|
throw new Error("Wasm interface verifier inspect boundary missing");
|
||||||
|
const report = wasmArtifactFixture();
|
||||||
|
const execution = wasmArtifactExecution(report);
|
||||||
|
const evidence = createAnalysisExecutionEvidence(
|
||||||
|
"inspect_wasm_artifact",
|
||||||
|
{ path: report.artifact.path },
|
||||||
|
execution,
|
||||||
|
"observed",
|
||||||
|
);
|
||||||
|
let response: unknown;
|
||||||
|
const inspect = runInNewContext(
|
||||||
|
`(${source.slice(declaration.start, declaration.end)})`,
|
||||||
|
{
|
||||||
|
assert,
|
||||||
|
client: { callTool: async () => response },
|
||||||
|
parseEvidence,
|
||||||
|
...(await verifierResults()),
|
||||||
|
},
|
||||||
|
) as (path: string) => Promise<unknown>;
|
||||||
|
response = { ...textReply(evidence), structuredContent: evidence };
|
||||||
|
await expect(inspect(report.artifact.path)).resolves.toEqual(evidence);
|
||||||
|
|
||||||
|
if (scenario === "transport disagreement") {
|
||||||
|
const different = wasmArtifactFixture();
|
||||||
|
different.limitations = ["Independent complete Wasm observation."];
|
||||||
|
const structured = createAnalysisExecutionEvidence(
|
||||||
|
"inspect_wasm_artifact",
|
||||||
|
{ path: different.artifact.path },
|
||||||
|
wasmArtifactExecution(different),
|
||||||
|
"observed",
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
parseEvidence(
|
||||||
|
toolContract("inspect_wasm_artifact").outputSchema.parse(structured),
|
||||||
|
),
|
||||||
|
).toEqual(structured);
|
||||||
|
response = { ...textReply(evidence), structuredContent: structured };
|
||||||
|
await expect(inspect(report.artifact.path)).rejects.toThrow(
|
||||||
|
"text and structured results disagree",
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const malformed = createEvidence(
|
||||||
|
execution.subject ?? undefined,
|
||||||
|
execution.provider,
|
||||||
|
{
|
||||||
|
operation: "inspect_wasm_artifact",
|
||||||
|
parameters: { path: report.artifact.path },
|
||||||
|
result: "authentic Evidence with an invalid Wasm result shape",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(parseEvidence(malformed)).toEqual(malformed);
|
||||||
|
response = { ...textReply(malformed), structuredContent: malformed };
|
||||||
|
await expect(inspect(report.artifact.path)).rejects.toThrow();
|
||||||
|
}
|
||||||
|
response = textReply(evidence);
|
||||||
|
await expect(inspect(report.artifact.path)).rejects.toThrow();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
it("preserves raw extraction manifest extensions for separate Evidence authentication", async () => {
|
||||||
|
const { requireMcpOutput } = await verifierResults();
|
||||||
|
const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left;
|
||||||
|
const inventory = artifactInventoryResultSchema.parse(
|
||||||
|
inventoryEvidence.normalized_result,
|
||||||
|
);
|
||||||
|
const extractionSemantic = {
|
||||||
|
source_manifest_id: inventory.manifest.manifest_id,
|
||||||
|
selected_occurrence_ids: [],
|
||||||
|
files_sha256: digestCanonicalValue([], "Artifact"),
|
||||||
|
output_root_alias: "$OUTPUT_ROOT",
|
||||||
|
};
|
||||||
|
const report = {
|
||||||
|
manifest: inventory.manifest,
|
||||||
|
extraction_manifest: {
|
||||||
|
extraction_id: `aex_${digestCanonicalValue(extractionSemantic, "Artifact")}`,
|
||||||
|
...extractionSemantic,
|
||||||
|
},
|
||||||
|
output_root: "/owned/output",
|
||||||
|
artifacts: [],
|
||||||
|
containment_verified: true,
|
||||||
|
cleanup: { attempted: true, verified: true, residual_paths: [] },
|
||||||
|
coverage: inventory.coverage,
|
||||||
|
provenance: inventory.provenance,
|
||||||
|
integrity_contradictions: inventory.integrity_contradictions,
|
||||||
|
limitations: [],
|
||||||
|
};
|
||||||
|
const evidenceFor = (result: unknown) =>
|
||||||
|
createEvidence(undefined, inventoryEvidence.provider, {
|
||||||
|
operation: "extract_artifact",
|
||||||
|
parameters: {},
|
||||||
|
result,
|
||||||
|
});
|
||||||
|
const evidence = evidenceFor(report);
|
||||||
|
const extendedResult = {
|
||||||
|
...report,
|
||||||
|
manifest: { ...report.manifest, vendor_note: "retained source extension" },
|
||||||
|
};
|
||||||
|
const tampered = { ...evidence, normalized_result: extendedResult };
|
||||||
|
const reply = (value: unknown) => ({
|
||||||
|
...textReply(value),
|
||||||
|
structuredContent: value,
|
||||||
|
});
|
||||||
|
// Shape projection discards the extension; it cannot authenticate its source ID.
|
||||||
|
expect(toolContract("extract_artifact").outputSchema.parse(tampered)).toEqual(
|
||||||
|
evidence,
|
||||||
|
);
|
||||||
|
const admittedTampered = requireMcpOutput(
|
||||||
|
reply(tampered),
|
||||||
|
"extract_artifact",
|
||||||
|
);
|
||||||
|
expect(admittedTampered).toEqual(tampered);
|
||||||
|
expect(() => parseEvidence(admittedTampered)).toThrow(/semantic identifier/u);
|
||||||
|
|
||||||
|
const authentic = evidenceFor(extendedResult);
|
||||||
|
const projected =
|
||||||
|
toolContract("extract_artifact").outputSchema.parse(authentic);
|
||||||
|
expect(projected).toEqual({ ...authentic, normalized_result: report });
|
||||||
|
const admittedAuthentic = requireMcpOutput(
|
||||||
|
reply(authentic),
|
||||||
|
"extract_artifact",
|
||||||
|
);
|
||||||
|
expect(admittedAuthentic).toEqual(authentic);
|
||||||
|
expect(parseEvidence(admittedAuthentic)).toEqual(authentic);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user