diff --git a/scripts/lib/ghidra-checkpoint-refusal.d.mts b/scripts/lib/ghidra-checkpoint-refusal.d.mts new file mode 100644 index 0000000000..9762e1aa2e --- /dev/null +++ b/scripts/lib/ghidra-checkpoint-refusal.d.mts @@ -0,0 +1,12 @@ +/** Require the CLI's explicit refusal instead of accepting an unrelated failure. */ +export function requireCheckpointCliRefusal( + operation: () => Promise, + expectedCode: string, +): Promise<{ readonly code: string; readonly [key: string]: unknown }>; + +/** Verify refusal before the caller restores its deliberately invalid manifest. */ +export function assertCheckpointRefusalPreservesManifest( + operation: () => Promise, + manifestPath: string, + expectedBytes: Uint8Array, +): Promise<{ readonly code: string; readonly [key: string]: unknown }>; diff --git a/scripts/lib/ghidra-checkpoint-refusal.mjs b/scripts/lib/ghidra-checkpoint-refusal.mjs new file mode 100644 index 0000000000..85a5a59fa8 --- /dev/null +++ b/scripts/lib/ghidra-checkpoint-refusal.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +/** Require the CLI's explicit refusal instead of accepting an unrelated failure. */ +export async function requireCheckpointCliRefusal(operation, expectedCode) { + let refusal; + try { + await operation(); + } catch (cause) { + assert.equal(cause.code, 1, cause.stderr); + refusal = JSON.parse(cause.stdout); + } + assert.ok(refusal, "CLI unexpectedly accepted unavailable checkpoint state"); + assert.equal(refusal.code, expectedCode, JSON.stringify(refusal)); + return refusal; +} + +/** Verify refusal before the caller restores its deliberately invalid manifest. */ +export async function assertCheckpointRefusalPreservesManifest( + operation, + manifestPath, + expectedBytes, +) { + const refusal = await requireCheckpointCliRefusal( + operation, + "configuration_invalid", + ); + assert.ok( + (await readFile(manifestPath)).equals(expectedBytes), + `Refused CLI command changed the checkpoint manifest: ${manifestPath}`, + ); + return refusal; +} diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 1e5d97c23c..ebdb475d0a 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -212,7 +212,12 @@ export async function verifyGhidraBoundaries( ); const names = await call("list_names"); const batchIdentity = await verifyGhidraBatchIdentity(call, names); - const instructionFlow = await verifyGhidraInstructionFlow(call, names, cli); + const instructionFlow = await verifyGhidraInstructionFlow( + call, + names, + cli, + target, + ); const leaf = names.find((item) => item.value.endsWith("rea_ghidra_inventory_leaf"), ); diff --git a/scripts/lib/real-ghidra-instruction-flow.d.mts b/scripts/lib/real-ghidra-instruction-flow.d.mts new file mode 100644 index 0000000000..70abf8964e --- /dev/null +++ b/scripts/lib/real-ghidra-instruction-flow.d.mts @@ -0,0 +1,19 @@ +import type { BinaryTarget } from "../../src/domain/binaryTargetTypes.js"; + +/** Check source-owned inventory controls using independently parsed target facts. */ +export function verifyGhidraInstructionFlow( + call: ( + operation: string, + parameters: Readonly>, + ) => Promise, + names: readonly { readonly address: string; readonly value: string }[], + cli: (command: string, address: string) => Promise, + target: Pick | null, +): Promise<{ + readonly return_instruction: string; + readonly decoded_pcode_return: true; + readonly return_cli_mcp_parity: true; + readonly explicit_x86_elf_controls: number; + readonly explicit_control_scope: string; + readonly generic_unknown_preserved: true; +}>; diff --git a/scripts/lib/real-ghidra-instruction-flow.mjs b/scripts/lib/real-ghidra-instruction-flow.mjs index 25bad8073e..bafbc13b2f 100644 --- a/scripts/lib/real-ghidra-instruction-flow.mjs +++ b/scripts/lib/real-ghidra-instruction-flow.mjs @@ -1,7 +1,13 @@ import assert from "node:assert/strict"; /** Verify decoded return evidence and distinct non-return flows on owned fixtures. */ -export async function verifyGhidraInstructionFlow(call, names, cli) { +export async function verifyGhidraInstructionFlow(call, names, cli, target) { + // inventory.c emits these controls only for an x86-64 ELF compilation. + // Provider name omission cannot establish whether that source branch applied. + const controlsRequired = + target !== null && + target.format === "elf" && + target.architecture === "x86_64"; const leaf = names.find((item) => item.value.endsWith("rea_ghidra_inventory_leaf"), ); @@ -31,6 +37,11 @@ export async function verifyGhidraInstructionFlow(call, names, cli) { let checked = 0; for (const [symbol, mnemonic, bytes] of controls) { const procedure = names.find((item) => item.value === symbol); + if (controlsRequired) + assert.ok( + procedure, + `Source fixture lacks expected decoder control ${symbol}`, + ); if (procedure === undefined) continue; const before = await call("resolve_containing_procedure", { address: procedure.address, @@ -72,9 +83,11 @@ export async function verifyGhidraInstructionFlow(call, names, cli) { return_cli_mcp_parity: true, explicit_x86_elf_controls: checked, explicit_control_scope: - checked === 0 - ? "Not present on this host-native fixture format/architecture" - : "x86-64 ELF", + target === null + ? "Unknown: source fixture format/architecture was not established" + : controlsRequired + ? "x86-64 ELF" + : "Not compiled for this fixture format/architecture", generic_unknown_preserved: true, }; } diff --git a/scripts/verify-real-ghidra-checkpoints.mjs b/scripts/verify-real-ghidra-checkpoints.mjs index 3b5788fce4..540c870d22 100644 --- a/scripts/verify-real-ghidra-checkpoints.mjs +++ b/scripts/verify-real-ghidra-checkpoints.mjs @@ -24,6 +24,10 @@ import { requireMcpOperationResult, requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; +import { + requireCheckpointCliRefusal, + assertCheckpointRefusalPreservesManifest, +} from "./lib/ghidra-checkpoint-refusal.mjs"; import { createVerifierRun, completeVerifierRun, @@ -109,18 +113,6 @@ async function cli(procedure, flags = [], path = target) { ); return JSON.parse(stdout); } -async function refuseCli(procedure, flags, expected) { - let refusal; - try { - await cli(procedure, flags); - } catch (cause) { - assert.equal(cause.code, 1, cause.stderr); - refusal = JSON.parse(cause.stdout); - } - assert.ok(refusal, "CLI unexpectedly accepted unavailable checkpoint state"); - assert.equal(refusal.code, expected, JSON.stringify(refusal)); - return refusal; -} async function connect(environment = {}) { const owner = createVerifierMcpOwner(); const client = new Client({ @@ -346,9 +338,8 @@ int main(void) { CheckpointRecord record = { 3, 11 }; return rea_checkpoint_leaf await phase( "an older immutable REA snapshot refuses the changed checkpoint", async () => { - const refusal = await refuseCli( - "rea_checkpoint_leaf", - ["--snapshot", snapshotPath], + const refusal = await requireCheckpointCliRefusal( + () => cli("rea_checkpoint_leaf", ["--snapshot", snapshotPath]), "evidence_integrity_mismatch", ); assert.ok(refusal.details.binding_mismatch.profile_digest); @@ -390,12 +381,16 @@ int main(void) { CheckpointRecord record = { 3, 11 }; return rea_checkpoint_leaf await phase( "target/profile mismatch is refused without replacing the saved state", async () => { + const attemptedBytes = Buffer.from( + JSON.stringify({ ...current, target_sha256: "0".repeat(64) }), + ); try { - await writeFile( + await writeFile(manifestPath, attemptedBytes); + await assertCheckpointRefusalPreservesManifest( + () => cli("rea_checkpoint_retained"), manifestPath, - JSON.stringify({ ...current, target_sha256: "0".repeat(64) }), + attemptedBytes, ); - await refuseCli("rea_checkpoint_retained", [], "configuration_invalid"); } finally { await writeFile(manifestPath, bytes); } @@ -405,9 +400,14 @@ int main(void) { CheckpointRecord record = { 3, 11 }; return rea_checkpoint_leaf await phase( "malformed manifest is refused without a fresh import", async () => { + const attemptedBytes = Buffer.from("{"); try { - await writeFile(manifestPath, "{"); - await refuseCli("rea_checkpoint_retained", [], "configuration_invalid"); + await writeFile(manifestPath, attemptedBytes); + await assertCheckpointRefusalPreservesManifest( + () => cli("rea_checkpoint_retained"), + manifestPath, + attemptedBytes, + ); } finally { await writeFile(manifestPath, bytes); } @@ -426,7 +426,10 @@ int main(void) { CheckpointRecord record = { 3, 11 }; return rea_checkpoint_leaf await copyFile(packed, backup); try { await writeFile(packed, "corrupted packed data"); - await refuseCli("rea_checkpoint_retained", [], "configuration_invalid"); + await requireCheckpointCliRefusal( + () => cli("rea_checkpoint_retained"), + "configuration_invalid", + ); } finally { await copyFile(backup, packed); } diff --git a/tests/conformance/providers/ghidra/checkpointRefusal.test.ts b/tests/conformance/providers/ghidra/checkpointRefusal.test.ts new file mode 100644 index 0000000000..05efc6abf2 --- /dev/null +++ b/tests/conformance/providers/ghidra/checkpointRefusal.test.ts @@ -0,0 +1,78 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { assertCheckpointRefusalPreservesManifest } from "../../../../scripts/lib/ghidra-checkpoint-refusal.mjs"; + +const checkpointFile = async (bytes: Uint8Array) => { + const root = await mkdtemp(join(tmpdir(), "rea-checkpoint-refusal-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "current.json"); + await writeFile(path, bytes); + return path; +}; + +const refusal = (code = "configuration_invalid", exitCode = 1) => + Object.assign(new Error("inert CLI refusal"), { + code: exitCode, + stdout: JSON.stringify({ code }), + stderr: "controlled CLI diagnostic", + }); + +it.each([ + { + name: "mismatched target identity", + bytes: Buffer.from(JSON.stringify({ target_sha256: "0".repeat(64) })), + }, + { name: "malformed manifest", bytes: Buffer.from("{") }, +])("checks refusal-time manifest bytes for $name", async ({ bytes }) => { + const path = await checkpointFile(bytes); + await expect( + assertCheckpointRefusalPreservesManifest( + async () => { + throw refusal(); + }, + path, + bytes, + ), + ).resolves.toEqual({ code: "configuration_invalid" }); + expect((await readFile(path)).equals(bytes)).toBe(true); + + await expect( + assertCheckpointRefusalPreservesManifest( + async () => { + // The same expected refusal must not hide replacement of the owned state. + await writeFile(path, JSON.stringify({ replaced: true })); + throw refusal(); + }, + path, + bytes, + ), + ).rejects.toThrow(/changed the checkpoint manifest/u); +}); + +it.each([ + { name: "unexpected success", operation: async () => undefined }, + { + name: "unrelated exit", + operation: async () => { + throw refusal("configuration_invalid", 2); + }, + }, + { + name: "different refusal", + operation: async () => { + throw refusal("capability_unavailable"); + }, + }, +])( + "rejects $name while checking manifest preservation", + async ({ operation }) => { + const bytes = Buffer.from("{"); + const path = await checkpointFile(bytes); + await expect( + assertCheckpointRefusalPreservesManifest(operation, path, bytes), + ).rejects.toThrow(); + expect((await readFile(path)).equals(bytes)).toBe(true); + }, +); diff --git a/tests/conformance/providers/ghidra/instructionFlowVerifier.test.ts b/tests/conformance/providers/ghidra/instructionFlowVerifier.test.ts new file mode 100644 index 0000000000..44714ed007 --- /dev/null +++ b/tests/conformance/providers/ghidra/instructionFlowVerifier.test.ts @@ -0,0 +1,118 @@ +import { expect, it } from "vitest"; +import { verifyGhidraInstructionFlow } from "../../../../scripts/lib/real-ghidra-instruction-flow.mjs"; + +const leaf = { address: "0x100", value: "rea_ghidra_inventory_leaf" }; +// Static decoder controls from inventory.c; no fixture bytes are executed. +const controls = [ + ["0x200", "rea_ghidra_flow_return", "RET", "c3"], + ["0x210", "rea_ghidra_flow_cleanup_return", "RET", "c21000"], + ["0x220", "rea_ghidra_flow_shared_tail", "JMP", "ebfe"], + ["0x230", "rea_ghidra_flow_trap", "UD2", "0f0b"], +] as const; + +const observedInstruction = ( + address: string, + mnemonic = "RET", + bytes = "c3", +) => ({ + address, + status: "decoded", + mnemonic, + bytes, + operands: [{ raw: "0x10" }], + flow: { + kind: mnemonic === "RET" ? "return" : "branch", + classification_evidence: [ + { source: "ghidra-listing-flow-type" }, + ...(mnemonic === "RET" + ? [{ source: "ghidra-instruction-pcode", value: "RETURN" }] + : []), + ], + }, + limitations: [], +}); +const instructions = new Map([ + [leaf.address, observedInstruction(leaf.address)], + ...controls.map( + ([address, , mnemonic, bytes]) => + [address, observedInstruction(address, mnemonic, bytes)] as const, + ), +]); +const call = async ( + operation: string, + input: Readonly>, +) => { + if (operation === "read_function_instructions") + return { instructions: [`${input.procedure}: fixture`] }; + if (operation === "inspect_native_instruction") + return input.address === "0x0" + ? { + status: "outside-memory", + flow: { kind: "unavailable", classification_evidence: [] }, + } + : instructions.get(input.address!); + if (operation === "resolve_native_call_targets") + return { status: "not-call" }; + if (operation === "resolve_containing_procedure") + return { address: input.address }; + throw new Error(`Unexpected fixture operation ${operation}`); +}; +const cli = async (_command: string, address: string) => + instructions.get(address); +const names = controls.map(([address, value]) => ({ address, value })); + +it("checks every retained inventory control for an independently parsed x86-64 ELF", async () => { + const result = await verifyGhidraInstructionFlow( + call, + [leaf, ...names], + cli, + { + format: "elf", + architecture: "x86_64", + }, + ); + expect(result.explicit_x86_elf_controls).toBe(4); + expect(result.decoded_pcode_return).toBe(true); +}); + +it.each(["all", "one"])( + "refuses %s omitted source controls on x86-64 ELF", + async (omission) => { + await expect( + verifyGhidraInstructionFlow( + call, + [leaf, ...(omission === "all" ? [] : names.slice(0, -1))], + cli, + { format: "elf", architecture: "x86_64" }, + ), + ).rejects.toThrow(/Source fixture lacks expected decoder control/u); + }, +); + +it.each([ + { format: "mach-o", architecture: "x86_64" }, + { format: "elf", architecture: "arm64" }, + { format: "pe", architecture: "x86_64" }, +] as const)( + "keeps controls non-applicable for $format/$architecture", + async (target) => { + const result = await verifyGhidraInstructionFlow(call, [leaf], cli, target); + expect(result.explicit_x86_elf_controls).toBe(0); + expect(result.generic_unknown_preserved).toBe(true); + expect(result.explicit_control_scope).toContain("Not compiled"); + }, +); + +it.each([0, 1, 4])( + "keeps explicit unknown applicability with %i observed controls", + async (observed) => { + const result = await verifyGhidraInstructionFlow( + call, + [leaf, ...names.slice(0, observed)], + cli, + null, + ); + expect(result.explicit_x86_elf_controls).toBe(observed); + expect(result.explicit_control_scope).toContain("Unknown"); + }, +);