mirror of
https://github.com/morluto/rea.git
synced 2026-11-17 04:04:25 +00:00
test: add critical provider workflow coverage
This commit is contained in:
1 parent
521139d4a5
commit
4716fa2ece
10 files changed
+363
-98
No files matched your search
@@ -1,5 +1,60 @@
|
||||
import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js";
|
||||
|
||||
/** Require truthful tri-state permissions from real Hopper memory regions. */
|
||||
export const requireTruthfulMemoryRegions = (segments) => {
|
||||
if (!Array.isArray(segments) || segments.length === 0) {
|
||||
throw new Error("list_segments returned no real memory regions");
|
||||
}
|
||||
const regions = segments.flatMap((segment) => [
|
||||
segment,
|
||||
...(Array.isArray(segment.sections) ? segment.sections : []),
|
||||
]);
|
||||
for (const region of regions) {
|
||||
for (const permission of ["readable", "writable", "executable"]) {
|
||||
if (![true, false, null].includes(region?.[permission])) {
|
||||
throw new Error(`list_segments omitted tri-state ${permission}`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
region.provenance !== "hopper-public-python-api" ||
|
||||
region.permissions?.available !== false ||
|
||||
typeof region.permissions.reason !== "string"
|
||||
) {
|
||||
throw new Error("list_segments omitted permission provenance");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Require correlated start and terminal progress from the real bridge. */
|
||||
export function requireBridgeProgress(updates) {
|
||||
const messages = updates.map(({ message }) => message);
|
||||
if (
|
||||
!messages.includes("Hopper bridge started request") ||
|
||||
!messages.includes("Hopper bridge completed request")
|
||||
)
|
||||
throw new Error(
|
||||
`The MCP client did not observe correlated Hopper bridge progress: ${JSON.stringify(updates)}`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Require one sanitized invalid-request diagnostic from the real bridge. */
|
||||
export function requireBridgeDiagnostic(chunks) {
|
||||
const diagnostics = chunks
|
||||
.join("")
|
||||
.split("\n")
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => JSON.parse(line));
|
||||
if (
|
||||
!diagnostics.some(
|
||||
(diagnostic) =>
|
||||
diagnostic.type === "bridge-diagnostic" &&
|
||||
diagnostic.category === "invalid_request" &&
|
||||
diagnostic.message === "Invalid Hopper bridge request",
|
||||
)
|
||||
)
|
||||
throw new Error("The MCP runtime omitted the correlated Hopper diagnostic");
|
||||
}
|
||||
|
||||
/** Verify named positive semantics from the source-owned C fixture. */
|
||||
export async function verifyRealHopperFixture({
|
||||
client,
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { join } from "node:path";
|
||||
import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
|
||||
@@ -142,24 +135,6 @@ const verifyPackagedArtifactExtraction = async ({
|
||||
(await readFile(join(outputRoot, "app/main.js"), "utf8")) !== "main();"
|
||||
)
|
||||
throw new Error("packaged artifact extraction CLI failed");
|
||||
if (
|
||||
(await readdir(workspace)).some((path) =>
|
||||
path.startsWith(".artifact-extraction.rea-"),
|
||||
)
|
||||
)
|
||||
throw new Error("packaged artifact extraction left staging paths");
|
||||
await rm(outputRoot, { recursive: true });
|
||||
if (await pathExists(outputRoot))
|
||||
throw new Error("packaged artifact extraction cleanup failed");
|
||||
};
|
||||
|
||||
const pathExists = async (path) => {
|
||||
try {
|
||||
await access(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const assertRoutedApplicationAnalysis = (analysis) => {
|
||||
|
||||
@@ -67,6 +67,29 @@ export function assertDossier(
|
||||
assertDossierMultiBlock(dossier, requireMultiBlock);
|
||||
}
|
||||
|
||||
export function assertDenseSwitchDossier(dossier, address) {
|
||||
assertDossier(dossier, {
|
||||
address,
|
||||
requireAssembly: true,
|
||||
requireMultiBlock: true,
|
||||
});
|
||||
const truncated = dossier.native_api.jump_tables.find(
|
||||
({ mappings_truncated: mappingsTruncated }) => mappingsTruncated,
|
||||
);
|
||||
if (
|
||||
truncated === undefined ||
|
||||
truncated.mappings.length !== 32 ||
|
||||
truncated.mappings.some(
|
||||
({ target_address: target, evidence }) =>
|
||||
!/^0x[0-9a-f]+$/u.test(target) ||
|
||||
!evidence.some(({ kind }) => kind === "jump-table"),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
`Ghidra dense jump-table truncation drifted: ${JSON.stringify(dossier.native_api.jump_tables)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function assertNativeApiBoundary(observed, requireJumpTable) {
|
||||
if (
|
||||
observed?.available !== true ||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "../dist/ghidra/GhidraInventoryValues.js";
|
||||
|
||||
import {
|
||||
assertDenseSwitchDossier,
|
||||
assertDossier,
|
||||
assertLocalProcedureInfo,
|
||||
assertProviderText,
|
||||
@@ -59,6 +60,10 @@ export async function verifyDebugFunctionOperations(
|
||||
procedures,
|
||||
"rea_ghidra_inventory_switch",
|
||||
);
|
||||
const denseSwitchProcedure = requireProcedure(
|
||||
procedures,
|
||||
"rea_ghidra_inventory_dense_switch",
|
||||
);
|
||||
const main = requireProcedure(procedures, "main");
|
||||
const entryString = strings.find(
|
||||
(item) => item.value === "REA_GHIDRA_INVENTORY_ENTRY",
|
||||
@@ -111,6 +116,14 @@ export async function verifyDebugFunctionOperations(
|
||||
const branchDossier = await analyzeBranchDossier(client, branch);
|
||||
const indirectDossier = await analyzeIndirectDossier(client, indirect);
|
||||
const switchDossier = await analyzeSwitchDossier(client, switchProcedure);
|
||||
const denseSwitchDossier = await functionCall(client, "analyze_function", {
|
||||
procedure: denseSwitchProcedure.address,
|
||||
include_assembly: true,
|
||||
limit: 500,
|
||||
max_pseudocode_chars: 100_000,
|
||||
max_instructions: 5_000,
|
||||
});
|
||||
assertDenseSwitchDossier(denseSwitchDossier, denseSwitchProcedure.address);
|
||||
assertIndirectDossier(indirectDossier, leaf.address);
|
||||
|
||||
const { cancellation, timeout } = await verifyRequestCancellationAndTimeout(
|
||||
@@ -124,6 +137,7 @@ export async function verifyDebugFunctionOperations(
|
||||
branch: dossierSummary(branchDossier),
|
||||
indirect: dossierSummary(indirectDossier),
|
||||
switch: dossierSummary(switchDossier),
|
||||
dense_switch: dossierSummary(denseSwitchDossier),
|
||||
instruction_window: {
|
||||
returned: instructionWindow.instructions.returned,
|
||||
truncated: instructionWindow.instructions.truncated,
|
||||
|
||||
@@ -127,6 +127,7 @@ try {
|
||||
architecture: "x86_64",
|
||||
});
|
||||
assertDebugFixture(debug);
|
||||
const nativeApiCli = await verifyNativeApiCli(debugPath);
|
||||
const stripped = await verifyTarget(strippedPath, "stripped", {
|
||||
format: "elf",
|
||||
architecture: "x86_64",
|
||||
@@ -166,6 +167,7 @@ try {
|
||||
summary(machObject),
|
||||
],
|
||||
malformed_target: "rejected-before-provider-start",
|
||||
native_api_cli: nativeApiCli,
|
||||
custom_target: custom === null ? null : summary(custom),
|
||||
cleanup: "complete",
|
||||
})}\n`,
|
||||
@@ -174,6 +176,52 @@ try {
|
||||
await rm(fixtureRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function verifyNativeApiCli(targetPath) {
|
||||
const { stdout } = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/rea.mjs",
|
||||
"inspect-native-api",
|
||||
targetPath,
|
||||
"rea_ghidra_inventory_dense_switch",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
maxBuffer: 2 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
const evidence = JSON.parse(stdout);
|
||||
const boundary = evidence.normalized_result?.boundary;
|
||||
const truncated = boundary?.jump_tables?.find(
|
||||
({ mappings_truncated: mappingsTruncated }) => mappingsTruncated === true,
|
||||
);
|
||||
const omittedQuestion =
|
||||
"Which additional data sources or targets were omitted";
|
||||
if (
|
||||
evidence.operation !== "inspect_native_api" ||
|
||||
evidence.provider?.id !== "rea-workflow" ||
|
||||
evidence.analysis_profile?.provider?.id !== "rea-workflow" ||
|
||||
boundary?.available !== true ||
|
||||
truncated?.mappings?.length !== 32 ||
|
||||
!evidence.normalized_result?.residual_unknowns?.some((question) =>
|
||||
question.includes(omittedQuestion),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
`The shipped inspect-native-api CLI did not preserve dense jump-table truncation: ${stdout}`,
|
||||
);
|
||||
return {
|
||||
operation: evidence.operation,
|
||||
provider: evidence.provider,
|
||||
mappings_returned: truncated.mappings.length,
|
||||
residual_unknowns: evidence.normalized_result.residual_unknowns.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyTarget(targetPath, variant, expectedTarget = null) {
|
||||
const parsedTarget = await parseBinaryTarget(targetPath);
|
||||
if (!parsedTarget.ok) throw parsedTarget.error;
|
||||
|
||||
@@ -16,8 +16,11 @@ import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js";
|
||||
import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js";
|
||||
import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs";
|
||||
import {
|
||||
requireBridgeDiagnostic,
|
||||
requireBridgeProgress,
|
||||
requireCurrentDocument,
|
||||
requireSafeDiagnostics,
|
||||
requireTruthfulMemoryRegions,
|
||||
verifyRealHopperFixture,
|
||||
} from "./lib/real-hopper-semantic.mjs";
|
||||
import {
|
||||
@@ -99,30 +102,6 @@ const requireOverview = (value, operation) => {
|
||||
return value;
|
||||
};
|
||||
|
||||
const requireTruthfulMemoryRegions = (segments) => {
|
||||
if (!Array.isArray(segments) || segments.length === 0) {
|
||||
throw new Error("list_segments returned no real memory regions");
|
||||
}
|
||||
const regions = segments.flatMap((segment) => [
|
||||
segment,
|
||||
...(Array.isArray(segment.sections) ? segment.sections : []),
|
||||
]);
|
||||
for (const region of regions) {
|
||||
for (const permission of ["readable", "writable", "executable"]) {
|
||||
if (![true, false, null].includes(region?.[permission])) {
|
||||
throw new Error(`list_segments omitted tri-state ${permission}`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
region.provenance !== "hopper-public-python-api" ||
|
||||
region.permissions?.available !== false ||
|
||||
typeof region.permissions.reason !== "string"
|
||||
) {
|
||||
throw new Error("list_segments omitted permission provenance");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const verifyRelationships = async (client, options, procedure) => {
|
||||
const related = {};
|
||||
for (const operation of ["procedure_callers", "procedure_callees"]) {
|
||||
@@ -312,7 +291,11 @@ try {
|
||||
);
|
||||
}
|
||||
|
||||
const options = { timeout };
|
||||
const progressUpdates = [];
|
||||
const options = {
|
||||
timeout,
|
||||
onprogress: (update) => progressUpdates.push(update),
|
||||
};
|
||||
const fullSessionStatus = () =>
|
||||
client.callTool(
|
||||
{ name: "binary_session", arguments: { detail: "full" } },
|
||||
@@ -343,6 +326,16 @@ try {
|
||||
{ name: "list_documents", arguments: {} },
|
||||
options,
|
||||
);
|
||||
requireBridgeProgress(progressUpdates);
|
||||
const rejectedProcedure = await client.callTool(
|
||||
{
|
||||
name: "procedure_info",
|
||||
arguments: { procedure: "__rea_missing_procedure__" },
|
||||
},
|
||||
options,
|
||||
);
|
||||
if (rejectedProcedure.isError !== true)
|
||||
throw new Error("The real Hopper bridge accepted an unknown procedure");
|
||||
const overview = await client.callTool(
|
||||
{ name: "binary_overview", arguments: {} },
|
||||
options,
|
||||
@@ -435,6 +428,7 @@ try {
|
||||
);
|
||||
}
|
||||
const diagnosticCount = requireSafeDiagnostics(stderrChunks);
|
||||
requireBridgeDiagnostic(stderrChunks);
|
||||
|
||||
const closed = await client.callTool(
|
||||
{ name: "close_binary", arguments: {} },
|
||||
@@ -460,6 +454,7 @@ try {
|
||||
bundledMcpRunning,
|
||||
stderrBytes,
|
||||
diagnosticCount,
|
||||
bridgeProgressUpdates: progressUpdates.length,
|
||||
dynamicSession: true,
|
||||
providerBinding,
|
||||
targets: [targetA, targetB, largeTarget],
|
||||
|
||||
Reference in new issue
Block a user