diff --git a/src/contracts/reverseEngineeringToolContracts.ts b/src/contracts/reverseEngineeringToolContracts.ts index 39d45fb899..8c31e875e3 100644 --- a/src/contracts/reverseEngineeringToolContracts.ts +++ b/src/contracts/reverseEngineeringToolContracts.ts @@ -39,7 +39,7 @@ export const REVERSE_ENGINEERING_TOOL_CONTRACTS = [ ...toolContractMetadata("execute_rizin_command"), kind: "native-provider", description: - "Run one caller-selected Rizin command on a local artifact. Commands/plugins can execute code, mutate files, access network and read outside the artifact; those external inputs are not Evidence-bound. Suppresses user settings/scripts but retains plugins. Raw stdout/stderr is authoritative, JSON is command-specific, and UTF-8 loss/truncation is explicit; output-limit termination leaves completion unknown. For raw images, asm_arch/asm_bits/asm_cpu select decoding and base_address maps offset 0; base_address also passes -n to skip loaders even on ELF, without REA confirming either. An admitted command retires shared binary analysis: close Ghidra explicitly first, then open_binary after effects settle. command_timeout_ms is per-phase and max_output_bytes bounds combined output (16 MiB default); both apply next call without restart and do not raise client receive limits or persistent debugger deadlines.", + "Run one caller-selected Rizin command on a local artifact. Commands/plugins can execute code, mutate files, access network and read outside the artifact; those external inputs are not Evidence-bound. Suppresses user settings/scripts but retains plugins. completion_status reports process completion; Rizin can reject a command and exit with code 0. Inspect stdout/stderr before treating empty output as absence. Raw stdout/stderr is authoritative, JSON is command-specific, and UTF-8 loss/truncation is explicit; output-limit termination leaves completion unknown. For raw images, asm_arch/asm_bits/asm_cpu select decoding and base_address maps offset 0; base_address also passes -n to skip loaders even on ELF, without REA confirming either. An admitted command retires shared binary analysis: close Ghidra explicitly first, then open_binary after effects settle. command_timeout_ms is per-phase and max_output_bytes bounds combined output (16 MiB default); both apply next call without restart and do not raise client receive limits or persistent debugger deadlines.", inputSchema: rizinInputSchema, outputSchema: evidenceResultOf(commandObservationSchema), examples: [ diff --git a/src/rizin/RizinCommand.ts b/src/rizin/RizinCommand.ts index 7ed8566054..9273cdf3b9 100644 --- a/src/rizin/RizinCommand.ts +++ b/src/rizin/RizinCommand.ts @@ -21,10 +21,11 @@ export const rizinCommand = (input: RizinInput): readonly string[] => [ absoluteCallerPath(input.path), ]; -/** What REA cannot attest about the decoder and map inputs it passed to Rizin. */ +/** What REA cannot attest about command outcomes and selected decoder/map inputs. */ export const rizinCommandLimitations = ( input: RizinInput, ): readonly string[] => [ + "completion_status reports process completion. Command validity remains unverified: Rizin can reject a command while returning exit_code 0. Inspect stdout and stderr before treating an empty response as absence.", ...(input.asm_arch === undefined && input.asm_bits === undefined && input.asm_cpu === undefined diff --git a/tests/acceptance/rizinCommandOutcomes.test.ts b/tests/acceptance/rizinCommandOutcomes.test.ts new file mode 100644 index 0000000000..f3c774af55 --- /dev/null +++ b/tests/acceptance/rizinCommandOutcomes.test.ts @@ -0,0 +1,89 @@ +import { createHash } from "node:crypto"; +import { chmod, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { REVERSE_ENGINEERING_TOOL_CONTRACTS } from "../../src/contracts/reverseEngineeringToolContracts.js"; +import { rizinCommandLimitations } from "../../src/rizin/RizinCommand.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +const outcomes = [ + { + command: "axffj", + stdout: "", + stderr: "ERROR: Command 'axffj' does not exist.\n", + }, + { command: "q", stdout: "", stderr: "" }, + { + command: "echo diagnostic", + stdout: "ERROR: Command 'axffj' does not exist.\n", + stderr: "WARNING: fixture warning\n", + }, +] as const; + +cliTest.skipIf(process.platform === "win32").for(outcomes)( + "preserves process facts and exposes command-validity uncertainty: $command", + async (outcome, { cli }) => { + const root = await createTestTempDirectory("rea-rizin-command-outcomes-"); + const provider = join(root, "rizin-fixture"); + const artifact = join(root, "artifact.bin"); + const content = "caller-selected inert artifact"; + await writeFile(artifact, content); + // Real Node subprocesses simulate producer output; they are not Rizin. + await writeFile( + provider, + `#!${process.execPath}\n` + + `const cases=${JSON.stringify(outcomes)};\n` + + "const selected=cases.find(item=>item.command===process.argv[process.argv.indexOf('-c')+1]);\n" + + "if(!selected)process.exit(99);\n" + + "process.stdout.write(selected.stdout);process.stderr.write(selected.stderr);\n", + ); + await chmod(provider, 0o755); + const environment = { REA_RIZIN_COMMAND: provider }; + const response = await cli.run({ + arguments: ["execute-rizin-command", artifact, outcome.command, "--json"], + environment, + }); + expect(response.exitCode, `${response.stdout}\n${response.stderr}`).toBe(0); + const { client, call } = await connectLocalToolsMcp(environment); + const definition = (await client.listTools()).tools.find( + ({ name }) => name === "execute_rizin_command", + ); + const mcp = await call("execute_rizin_command", { + path: artifact, + command: outcome.command, + }); + expect(mcp.isError, JSON.stringify(mcp.content)).not.toBe(true); + const contract = REVERSE_ENGINEERING_TOOL_CONTRACTS.find( + ({ name }) => name === "execute_rizin_command", + ); + if (!contract) throw new Error("Missing execute_rizin_command contract"); + const limitations = rizinCommandLimitations({ + path: artifact, + command: outcome.command, + }); + // Even unqualified commands must disclose the provider validity gap. + expect(limitations).not.toHaveLength(0); + for (const raw of [response.json, mcp.structuredContent]) { + const evidence = contract.outputSchema.parse(raw); + expect(evidence.normalized_result).toMatchObject({ + stdout: outcome.stdout, + stderr: outcome.stderr, + exit_code: 0, + completion_status: "complete", + output_truncated: false, + artifact_changed: false, + }); + expect(evidence.raw_result).toEqual(evidence.normalized_result); + expect(evidence.subject?.digest.sha256).toBe( + createHash("sha256").update(content).digest("hex"), + ); + expect(evidence.parameters.command).toBe(outcome.command); + expect(evidence.limitations).toEqual( + expect.arrayContaining([...limitations]), + ); + } + expect(definition?.description).toBe(contract.description); + }, +); diff --git a/tests/composition/reverseEngineering.test.ts b/tests/composition/reverseEngineering.test.ts index ff7fc18e01..e5480ff776 100644 --- a/tests/composition/reverseEngineering.test.ts +++ b/tests/composition/reverseEngineering.test.ts @@ -404,7 +404,11 @@ it("marks signal-terminated provider commands as unknown completion", async () = /files.*network.*outside the selected artifact.*not bound to Evidence/iu, ), ); - expect(ordinaryExit.value.limitations).toHaveLength(4); + expect(ordinaryExit.value.limitations).toEqual( + expect.arrayContaining([ + ...rizinCommandLimitations({ path, command: "iI" }), + ]), + ); } } finally { await rm(directory, { recursive: true, force: true }); @@ -476,7 +480,11 @@ describe("Rizin command service", () => { /files.*network.*outside the selected artifact.*not bound to Evidence/iu, ), ); - expect(result.value.limitations).toHaveLength(4); + expect(result.value.limitations).toEqual( + expect.arrayContaining([ + ...rizinCommandLimitations({ path: callerPath, command }), + ]), + ); expect(result.value.parameters).toMatchObject({ path: callerPath, command, @@ -611,15 +619,15 @@ it("passes decoder and map inputs to Rizin before the path and records them", as expect(result.value.limitations).toEqual( expect.arrayContaining([...rizinCommandLimitations(input)]), ); - // One statement for the decoder inputs and one for the raw map. - expect(rizinCommandLimitations(input)).toHaveLength(2); + // Command validity stays unverified independently of decoder/map inputs. + expect(rizinCommandLimitations(input)).toHaveLength(3); expect( rizinCommandLimitations({ path, command: "pi 1", asm_bits: 16 }), - ).toHaveLength(1); + ).toHaveLength(2); expect( rizinCommandLimitations({ path, command: "pi 1", base_address: "0x0" }), - ).toHaveLength(1); - expect(rizinCommandLimitations({ path, command: "pi 1" })).toEqual([]); + ).toHaveLength(2); + expect(rizinCommandLimitations({ path, command: "pi 1" })).toHaveLength(1); } finally { await rm(directory, { recursive: true, force: true }); }