mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 00:36:35 +00:00
morluto
1a0e2c77f2
* fix(ghidra): qualify selected project call coverage * test(ghidra): retain verifier MCP cleanup ownership * test(ghidra): document trusted verifier compiler selection * test(ghidra): document owned verifier exit marker * test(ghidra): remove ineffective scan directives * fix(ghidra): supervise project coverage verification * fix(ghidra): supervise verifier compiler process
422 lines
14 KiB
JavaScript
422 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import assert from "node:assert/strict";
|
|
import { createHash } from "node:crypto";
|
|
import {
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { realpathSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { isAbsolute, join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import {
|
|
OwnedCommandFailure,
|
|
runOwnedCommand,
|
|
} from "../dist/process/OwnedCommand.js";
|
|
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
|
import { Client } from "@modelcontextprotocol/client";
|
|
import { createAnalysisProfile } from "../dist/domain/analysisProfile.js";
|
|
import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js";
|
|
import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-inspection.mjs";
|
|
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
|
import {
|
|
createVerifierMcpOwner,
|
|
VerifierStdioTransport,
|
|
} from "./lib/verifier-mcp-owner.mjs";
|
|
import {
|
|
requireMcpOutput,
|
|
requireMcpToolError,
|
|
} from "./lib/mcp-verifier-results.mjs";
|
|
|
|
export async function verifyProjectCoverage(run) {
|
|
const installDir = process.env.GHIDRA_INSTALL_DIR;
|
|
if (installDir === undefined || !isAbsolute(installDir))
|
|
throw new Error(
|
|
"Set GHIDRA_INSTALL_DIR to an absolute Ghidra installation path.",
|
|
);
|
|
const installation = await inspectVerifierGhidraInstallation({
|
|
environment: process.env,
|
|
installDir,
|
|
...(process.env.JAVA_HOME === undefined
|
|
? {}
|
|
: { javaHome: process.env.JAVA_HOME }),
|
|
});
|
|
if (installation.status !== "available")
|
|
throw new Error(`Ghidra is unavailable: ${JSON.stringify(installation)}`);
|
|
if (process.platform === "win32")
|
|
throw new Error(
|
|
"REA selected Ghidra projects are supported on POSIX hosts.",
|
|
);
|
|
const scriptRoot = fileURLToPath(
|
|
new URL("../tests/conformance/ghidra/", import.meta.url),
|
|
);
|
|
const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url));
|
|
const workspace = await mkdtemp(
|
|
join(tmpdir(), "rea-ghidra-project-coverage-"),
|
|
);
|
|
const connections = createVerifierMcpOwner();
|
|
const target = join(workspace, "fixture");
|
|
const projects = join(workspace, "projects");
|
|
const env = {
|
|
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
|
JAVA_HOME: installation.javaHome,
|
|
GHIDRA_INSTALL_DIR: installation.installDir,
|
|
REA_MCP_OUTPUT_SCHEMA_PROFILE: "compact",
|
|
REA_LOG_LEVEL: "silent",
|
|
REA_PROCESS_RUN_ID: run.run_id,
|
|
};
|
|
const hash = (data) => createHash("sha256").update(data).digest("hex");
|
|
async function manifest(directory) {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
const files = await Promise.all(
|
|
entries.map(async (entry) => {
|
|
const path = join(directory, entry.name);
|
|
return entry.isDirectory()
|
|
? manifest(path)
|
|
: entry.isFile()
|
|
? [[path, hash(await readFile(path))]]
|
|
: [];
|
|
}),
|
|
);
|
|
return files.flat().sort((a, b) => a[0].localeCompare(b[0]));
|
|
}
|
|
async function connect(environment) {
|
|
const client = new Client({
|
|
name: "selected-project-coverage-verifier",
|
|
version: "1",
|
|
});
|
|
const transport = new VerifierStdioTransport({
|
|
command: process.execPath,
|
|
args: [entrypoint, "mcp"],
|
|
env: environment,
|
|
stderr: "pipe",
|
|
});
|
|
return connections.open(client, transport);
|
|
}
|
|
async function call(connection, name, args = {}) {
|
|
const reply = await connection.client.callTool(
|
|
{ name, arguments: args },
|
|
{ timeout: 180000, toolDefinition: connection.definitions.get(name) },
|
|
);
|
|
return requireMcpOutput(reply, name);
|
|
}
|
|
let commandCleanupIncomplete = false;
|
|
const command = async (executable, args, timeoutMs) => {
|
|
try {
|
|
return await runOwnedCommand(
|
|
{
|
|
command: executable,
|
|
arguments: args,
|
|
env,
|
|
runId: run.run_id,
|
|
expectedCommand: null,
|
|
},
|
|
{ timeoutMs, diagnosticBytes: 16 * 1024 * 1024 },
|
|
);
|
|
} catch (cause) {
|
|
commandCleanupIncomplete =
|
|
cause instanceof OwnedCommandFailure &&
|
|
cause.cleanupOwner !== undefined;
|
|
throw cause;
|
|
}
|
|
};
|
|
const result = await withVerifierCleanup(async () => {
|
|
await mkdir(projects);
|
|
// REA_CC is the invoking developer's explicitly selected compiler, not target input.
|
|
await command(
|
|
process.env.REA_CC ?? "cc",
|
|
[
|
|
"-O0",
|
|
"-g",
|
|
"-fno-inline",
|
|
"-fno-optimize-sibling-calls",
|
|
join(scriptRoot, "selected-project-coverage.c"),
|
|
"-o",
|
|
target,
|
|
],
|
|
120000,
|
|
);
|
|
for (const name of ["partial", "analyzed"]) {
|
|
const args = [projects, name, "-import", target];
|
|
if (name === "partial")
|
|
args.push(
|
|
"-noanalysis",
|
|
"-scriptPath",
|
|
scriptRoot,
|
|
"-postScript",
|
|
"ReaSelectedProjectSeeds.java",
|
|
);
|
|
const result = await command(
|
|
installation.analyzeHeadlessPath,
|
|
args,
|
|
180000,
|
|
);
|
|
if (name === "partial")
|
|
for (const seed of [
|
|
"rea_project_leaf",
|
|
"rea_project_caller",
|
|
"rea_project_other",
|
|
])
|
|
assert.match(
|
|
result.stdout.text,
|
|
new RegExp(`REA_PROJECT_SEED ${seed} `),
|
|
);
|
|
}
|
|
const before = {
|
|
target: hash(await readFile(target)),
|
|
projects: await manifest(projects),
|
|
};
|
|
const report = [];
|
|
for (const mode of ["fresh", "partial", "analyzed"]) {
|
|
const selected = mode !== "fresh";
|
|
const connection = await connect({
|
|
...env,
|
|
...(selected
|
|
? {
|
|
REA_GHIDRA_PROJECT_PATH: join(projects, `${mode}.gpr`),
|
|
REA_GHIDRA_PROGRAM_NAME: "fixture",
|
|
}
|
|
: {}),
|
|
});
|
|
const bundlePath = join(workspace, `${mode}.json`);
|
|
const paths = [];
|
|
await withVerifierCleanup(async () => {
|
|
await call(connection, "open_binary", {
|
|
path: target,
|
|
provider_id: "ghidra",
|
|
});
|
|
const inventory = (
|
|
await call(connection, "list_procedures", {
|
|
detail: "complete",
|
|
})
|
|
).normalized_result;
|
|
const entry = (name) => {
|
|
const matches = inventory.filter(
|
|
(item) => item.value === name || item.value === `_${name}`,
|
|
);
|
|
assert.equal(matches.length, 1, `Expected one entry for ${name}`);
|
|
return matches[0].address;
|
|
};
|
|
const leaf = entry("rea_project_leaf"),
|
|
caller = entry("rea_project_caller"),
|
|
other = entry("rea_project_other");
|
|
const direct = await call(connection, "procedure_callees", {
|
|
procedure: caller,
|
|
});
|
|
assert.equal(
|
|
direct.relationship_coverage.status,
|
|
selected ? "unknown" : "complete",
|
|
);
|
|
assert.equal(
|
|
direct.normalized_result.includes(leaf),
|
|
mode !== "partial",
|
|
);
|
|
assert.equal(
|
|
direct.analysis_profile.parameters
|
|
.selected_project_relationship_coverage,
|
|
selected ? "prior-analysis-unknown-v1" : undefined,
|
|
);
|
|
for (const [start, goal, direction] of [
|
|
[caller, leaf, "forward"],
|
|
[leaf, caller, "backward"],
|
|
[other, caller, "backward"],
|
|
]) {
|
|
const evidence = await call(connection, "trace_call_path", {
|
|
start,
|
|
goal,
|
|
direction,
|
|
});
|
|
paths.push(evidence);
|
|
}
|
|
assert.equal(
|
|
paths[0].normalized_result.goal_status,
|
|
mode === "partial" ? "unknown" : "reached",
|
|
);
|
|
assert.equal(
|
|
paths[1].normalized_result.goal_status,
|
|
mode === "partial" ? "unknown" : "reached",
|
|
);
|
|
assert.equal(paths[2].normalized_result.goal_status, "unknown");
|
|
if (!selected) {
|
|
assert.equal(
|
|
paths[2].normalized_result.search_scope.observed_goal_status,
|
|
"not_reached",
|
|
);
|
|
assert.equal(
|
|
paths[2].normalized_result.search_scope.discovery,
|
|
"unknown",
|
|
);
|
|
}
|
|
if (selected)
|
|
for (const evidence of paths)
|
|
assert.ok(
|
|
Object.values(
|
|
evidence.normalized_result.relationship_coverage,
|
|
).some(
|
|
(coverage) =>
|
|
coverage.status === "unknown" &&
|
|
/prior analysis completeness is unknown/u.test(
|
|
coverage.reason,
|
|
),
|
|
),
|
|
);
|
|
if (mode === "analyzed") {
|
|
const dossier = await call(connection, "analyze_function", {
|
|
procedure: entry("rea_project_callback"),
|
|
detail: "complete",
|
|
});
|
|
assert.equal(dossier.relationship_coverage.status, "unknown");
|
|
assert.match(
|
|
dossier.relationship_coverage.reason,
|
|
/decoded call\(s\) without destinations/u,
|
|
);
|
|
assert.match(
|
|
dossier.relationship_coverage.reason,
|
|
/prior analysis completeness is unknown/u,
|
|
);
|
|
const callbackPath = await call(connection, "trace_call_path", {
|
|
start: entry("rea_project_callback"),
|
|
goal: leaf,
|
|
direction: "forward",
|
|
});
|
|
const coverages = Object.values(
|
|
callbackPath.normalized_result.relationship_coverage,
|
|
);
|
|
assert.ok(
|
|
coverages.some(
|
|
(coverage) =>
|
|
coverage.status === "partial" &&
|
|
coverage.limitations.some((text) =>
|
|
/decoded call\(s\) without destinations/u.test(text),
|
|
) &&
|
|
coverage.limitations.some((text) =>
|
|
/prior analysis completeness is unknown/u.test(text),
|
|
),
|
|
),
|
|
);
|
|
}
|
|
await call(connection, "export_evidence_bundle", { path: bundlePath });
|
|
if (mode === "partial") {
|
|
const snapshotPath = join(workspace, "partial-snapshot.json");
|
|
await call(connection, "close_binary", {
|
|
snapshot_path: snapshotPath,
|
|
});
|
|
const snapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
|
|
assert.equal(
|
|
snapshot.binding.analysis_profile.parameters
|
|
.selected_project_relationship_coverage,
|
|
"prior-analysis-unknown-v1",
|
|
);
|
|
const profile = snapshot.binding.analysis_profile;
|
|
const oldParameters = { ...profile.parameters };
|
|
delete oldParameters.selected_project_relationship_coverage;
|
|
const stalePath = join(workspace, "partial-old-profile.json");
|
|
await writeFile(
|
|
stalePath,
|
|
JSON.stringify({
|
|
...snapshot,
|
|
binding: {
|
|
...snapshot.binding,
|
|
analysis_profile: createAnalysisProfile(
|
|
profile.provider,
|
|
oldParameters,
|
|
),
|
|
},
|
|
entries: [],
|
|
workflow_entries: [],
|
|
evidence_bundle: createEvidenceBundle([]),
|
|
}),
|
|
);
|
|
const staleBefore = await readFile(stalePath);
|
|
const refused = await connection.client.callTool({
|
|
name: "open_binary",
|
|
arguments: {
|
|
path: target,
|
|
provider_id: "ghidra",
|
|
snapshot_path: stalePath,
|
|
},
|
|
});
|
|
const error = requireMcpToolError(refused);
|
|
assert.equal(error.code, "evidence_integrity_mismatch");
|
|
assert.deepEqual(error.details.binding_mismatch.profile_digest, {
|
|
snapshot: createAnalysisProfile(profile.provider, oldParameters)
|
|
.digest,
|
|
selected: profile.digest,
|
|
});
|
|
assert.deepEqual(await readFile(stalePath), staleBefore);
|
|
await call(connection, "open_binary", {
|
|
path: target,
|
|
provider_id: "ghidra",
|
|
snapshot_path: snapshotPath,
|
|
});
|
|
const replay = await call(connection, "trace_call_path", {
|
|
start: caller,
|
|
goal: leaf,
|
|
direction: "forward",
|
|
});
|
|
assert.deepEqual(
|
|
replay.normalized_result,
|
|
paths[0].normalized_result,
|
|
);
|
|
await call(connection, "close_binary");
|
|
} else await call(connection, "close_binary");
|
|
}, [() => connections.closeConnection(connection)]);
|
|
const offline = await connect(env);
|
|
await withVerifierCleanup(async () => {
|
|
await call(offline, "import_evidence_bundle", { path: bundlePath });
|
|
const records = (
|
|
await call(offline, "get_evidence_bundle", { detail: "complete" })
|
|
).result.records;
|
|
for (const evidence of paths) {
|
|
const saved = records.find(
|
|
(record) => record.evidence_id === evidence.evidence_id,
|
|
);
|
|
assert.ok(saved);
|
|
assert.deepEqual(saved.normalized_result, evidence.normalized_result);
|
|
assert.deepEqual(saved.limitations, evidence.limitations);
|
|
}
|
|
}, [() => connections.closeConnection(offline)]);
|
|
report.push({
|
|
mode,
|
|
statuses: paths.map(
|
|
(evidence) => evidence.normalized_result.goal_status,
|
|
),
|
|
});
|
|
}
|
|
assert.deepEqual(
|
|
{
|
|
target: hash(await readFile(target)),
|
|
projects: await manifest(projects),
|
|
},
|
|
before,
|
|
);
|
|
return {
|
|
ok: true,
|
|
provider: installation.providerVersion,
|
|
results: report,
|
|
};
|
|
}, [
|
|
async () => {
|
|
await connections.close();
|
|
if (!commandCleanupIncomplete)
|
|
await rm(workspace, { recursive: true, force: true });
|
|
},
|
|
]);
|
|
return result;
|
|
}
|
|
|
|
if (
|
|
process.argv[1] !== undefined &&
|
|
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
|
|
) {
|
|
const run = createVerifierRun();
|
|
const report = await verifyProjectCoverage(run);
|
|
await completeVerifierRun(run);
|
|
process.stdout.write(`${JSON.stringify(report)}\n`);
|
|
}
|