diff --git a/docs/native-investigation.md b/docs/native-investigation.md index e2a3f6ef06..839a029024 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -175,6 +175,46 @@ remains valid. This tool has no dedicated CLI command; the existing `decompile` command remains the single-procedure CLI operation. +`decompile_all` (`rea decompile-all --output `) writes one +pseudocode file per analyzed function and returns only a manifest: counts, the +output directory, capped failure reasons, provider identity, and the +analysis-profile digest. The manifest and `decompile-index.json` do not contain +pseudocode. Each file records the provider name, canonical address, inclusive +body ranges when `procedure_info` supplies them, and the `procedure_info` and +pseudocode limitations for that function. Header values are abbreviated to +2,000 characters per value and the joined limitation line to 4,000 characters; +the index retains the complete selected names, addresses, provider facts and +caveats. Failure details retain at most 32 rows within 2,048 JSON-encoded +UTF-8 bytes; names may be omitted to keep complete addresses and reasons. The +index retains every failure and its complete observed name. +Each function's `decompile-index.json` record, whatever its status, keeps the +complete limitations of the observations it used. The returned Evidence keeps +at most eight first-observed distinct per-function provider limitations within an 8,000-byte JSON-encoded UTF-8 text budget and counts omitted observations. Repeated omitted text is counted per observation, not as a distinct caveat. This summary does not retain every omitted string in memory. A fresh output directory must not already exist. `--resume` +continues an export for the same target SHA-256, provider, analysis profile, +and layout; it requires the directory to exist, and an export without a known +target or profile digest cannot be resumed. Resume skips files whose bytes still match the last write the journal +recorded, even after a later read failure, tries +functions it has not attempted before retrying recorded failures, ignores a +journal line torn by an interrupted write, and never overwrites a conflicting +file. File reservations compare ASCII paths without case, so distinct address +spaces remain distinct on case-insensitive filesystems. A resumed journal with +conflicting reservations is refused. One function's decompiler error or deadline is counted and does not stop +the others. A function that `procedure_info` classifies as external, such as an +import, is counted as `external` and recorded with that status and the body +fact as observed. It gets no file, no pseudocode is requested, and a resume +skips it. A function whose classification is unknown is decompiled as usual; +a provider that returns null pseudocode for it is reported as `no_pseudocode`. Inventory rows without an address are counted in the limitations. `--layout dwarf` +reports that compilation-unit paths are unavailable and still names files by +address. Filters are a case-sensitive name substring, a case-sensitive name +prefix, an inclusive address range (including uppercase hexadecimal input), and a positive safe-integer attempt limit. The limit does not allocate space in proportion to its value. Cancellation leaves +files already written in place. One run at a time holds an output directory, +identified by its device and inode, through export finalization and any retained cleanup retry; another +run on the same directory is refused with the holder's process ID, and a +crashed run can be recovered by the next run. Failed descriptor or staging cleanup remains owned by the session; close the target to retry it. Initialization cleanup preserves unexpected entries rather than recursively removing the directory. Resume hashes existing files in bounded chunks, and index generation reuses committed journal facts already held by the export owner. After five consecutive functions whose files +could not be written or read, for example on a full disk, the export stops with +an error instead of decompiling the rest. Files and records already committed +stay, `decompile-index.json` is still written, and a later resume continues. + Type inspection selects one exact database pathname or typed data address. Struct/union fields, enums, pointers, arrays, typedefs, size, alignment, packing and bitfields retain database authority. Child types use exact IDs for further diff --git a/docs/verification/native-decompile-directory-2026-10-23.md b/docs/verification/native-decompile-directory-2026-10-23.md new file mode 100644 index 0000000000..9f9ff5d7fe --- /dev/null +++ b/docs/verification/native-decompile-directory-2026-10-23.md @@ -0,0 +1,223 @@ +# Native directory decompilation verification — 2026-10-23 + +The completed analyst task is to export the bound provider's function inventory +as one local pseudocode file per non-external function, with names, body ranges, +provider/profile identity and the limitations of each used observation. The +agent receives a manifest rather than the pseudocode. A function decompilation +failure remains a recorded partial outcome and does not end the export. + +## Artifact, provider and correctness + +The real target was the installed ELF x86-64 `/usr/bin/sha256sum` (60,368 bytes), +SHA-256 `6cd7c6bfc81d645ba13b927e31651a1466092a28ed0bd2632e82f8b27882b25e`. +It was read and analyzed statically, never executed. Both variants used the +existing Ghidra 12.1.4 installation and JDK 21.0.12.1+1, Node 24.22.1, the locked +npm dependencies, `GHIDRA_HEADLESS_MAXMEM=1G` and +`GHIDRA_HEADLESS_JAVA_OPTIONS=-XX:ActiveProcessorCount=2`. +The upstream analysis-profile digest was +`ae80ece8c83379eddabbdf379bbffca5da84c7df7fa7c159f45fa810468f85d9`. + +Baseline main was `5d2c8c2b78d2552da0e13a11d3e2678efb69810d`. +The feature branch integrates original PR head +`d3a207038cfc00d58a97b86b06af1368839bec99` on that base, with the maintenance +changes described below. The frozen integration is `c566cace04d24f8ccb3d54353ababf95b96c8830`, +including refreshed main `bded9c628de23643aab81f98959b6a473318a240`. The review fixes were verified on `d3e0e96821a09a447785d100f3d85139b6e77e96`. Each variant ran through the public production +`scripts/rea.mjs mcp` entrypoint and the installed MCP SDK client. + +The baseline called `list_procedures`, then `procedure_info` for each addressed +entry and `procedure_pseudo_code` for each non-external entry. A local verifier +used the returned Evidence to materialize the same function files and retain an +index. It reused the feature's pure file-header renderer to keep file encoding +equal; baseline analysis itself used only the existing public primitive tools. +The existing `batch_decompile` item format does not retain per-function caveat +associations; its workflow limitations aggregate across items. It would not +provide equivalent per-function caveat provenance here, so the baseline uses +primitive observations. The feature called `decompile_all` once per output directory. Both retained the +complete MCP responses locally. A second export used a new output directory in +the same session. The feature additionally resumed the verified warm export. + +Both cold and warm exports examined the same 298 provider inventory entries: +236 written, 61 external, one failed. Addresses, statuses, body ranges, function +names, pseudocode SHA-256 and complete emitted-file SHA-256 matched for every +entry in both rounds. The target's bytes stayed unchanged. This proves export +preservation of the provider's observed results; it does not establish that the +provider found every actual function or that its pseudocode is original source. + +Ghidra failed at `0x10a510` with `Overlapping input varnodes`; the failure code +was `execution_failure`. The export still wrote 68 functions at later numeric +addresses. Resume skipped every successful/external entry and retried the one +failed entry without rewriting the successful files. Complete per-function +caveats stayed in the index, and no pseudocode was returned in the manifest. + +## Measurements + +| Completed workflow | Before: public primitives + local files | After: directory workflow | +| ---------------------------------------------- | --------------------------------------: | ------------------------: | +| Cold export: MCP calls | 536 | 1 | +| Cold export: request + response token estimate | 1,375,776 | 3,800 | +| Cold export: elapsed | 50.940 s | 39.155 s | +| Cold export: sampled family RSS peak | 1754.5 MiB | 1709.3 MiB | +| Warm export: MCP calls | 536 | 1 | +| Warm export: request + response token estimate | 1,377,486 | 3,800 | +| Warm export: elapsed | 8.566 s | 8.058 s | +| Warm export: sampled family RSS peak | 1779.4 MiB | 1710.6 MiB | + +The identical two-export task, including the full catalog discovery, open and +close, used **3,723,191 → 981,913 estimated tokens (73.63% fewer)**. +The extra verified-resume call is excluded from that comparison. The catalog +itself grew from 969,665 to +974,049 estimated tokens. All 220 existing tool definitions +were identical; only `decompile_all` was added. The per-export request/reply +reduction is about 99.72%; applying that percentage to the entire task would +overstate the benefit. A client which injects the entire catalog still pays its +large fixed context cost. + +The full verifier process family took 85.697 → 62.034 s elapsed, +111.166 → 68.185 s waited CPU, and +1779.4 → 1713.7 MiB sampled peak RSS. The after run additionally includes +verified resume (0.266 s). These compare the retained baseline with the final review-fix run; +**no general CPU or elapsed-time speedup is claimed**. Earlier maintenance +trials ranged 50–83 s elapsed and 65–118 s CPU; they used earlier source +revisions and are retained separately. Full-run figures include SDK, verifier +formatting/hash checks and token estimation as well as REA and its provider. +Per-phase sampled CPU was 70.19 → 43.16 s cold and +6.15 → 4.42 s warm, subject to delayed child accounting. The +integration decision rests on equivalent results with far fewer agent-facing +calls and estimated context, plus bounded auxiliary accumulators. + +Cold/warm here describe primitive analysis reuse inside one provider session. +Host filesystem caches were not flushed, and the baseline ran first. This is a +paired functional measurement, not a statistical speedup claim. Provider startup +and host load varied: the independent CLI runs below took longer than the MCP +runs. No claim is made about cold OS caches or other binaries. + +Token counts use `tokenx` 1.3.0 estimates of actual tool arguments and complete +MCP call results, including both textual and structured projections. They are +not model-specific BPE counts or actual model usage. The completed-task total +also includes the actual `tools/list` response, open and close calls, and both +exports. MCP handshake framing, orchestrator prompts, reasoning, subsequent +agent reading of files and unknown model tokenizer costs are unmeasured. + +Process-family RSS was sampled from `/proc` every 50 ms. Shared pages can be +counted more than once, and short peaks can be missed. Full-run CPU uses waited +child usage; per-phase CPU is a sampled `/proc` delta and can include delayed +accounting of exited descendants. Limits were two CPU cores, a 2 GiB V8 heap per +Node process, 1 GiB configured Java heap, 6 GiB sampled family RSS, 300 sampled +family CPU seconds and 600 elapsed seconds. These are verifier controls, not a +kernel memory limit. All reported completed runs exited zero without a resource +stop. No reduction in Java's analysis memory is claimed. + +The first baseline verifier attempt stopped at the real per-function Ghidra +error because its harness initially required every primitive call to succeed. +It took 74.325 s elapsed and 101.569 s waited CPU, with no resource stop. The +completed baseline was corrected to retain that same failure and continue, +matching the requested partial-result semantics. The failed trial is retained +separately and is excluded from the paired completed-export table; no provider +fix or disappearance of that failure is claimed. + +## Ownership, reuse and resource limits + +The export owner parses a resumed journal once and updates its latest records +only after append and sync complete. The shared function-record schema owns +path/status validation for both journal and index; the append retains the +parsed snapshot, including nested fields, rather than the caller's mutable input. Index generation consumes those committed +facts instead of reading and parsing the whole journal again. Resume identity +covers target SHA-256, provider identity/version, analysis-profile digest and +layout. Unknown target/profile digests prevent resume. Files are checked afresh +with the shared stable-file hash owner and bounded reads, rather than trusting +an old checksum as current. `decompile_all` remains ineligible for snapshot +replay: filesystem publication must run for the actual request. + +Journal descriptors, verified staging areas and the directory lease have one +cleanup owner. Failed close/removal retains that owner under the session; the +lease is released after child cleanup and initialization cleanup finish. +Initialization removes verified owned entries rather than recursively deleting +unexpected files. Function path reservations use case-folded ASCII lookup +keys while preserving the original address identity and chosen filename; +collisions receive a stable address digest suffix. Resume refuses conflicting +legacy reservations. These cases are exercised with the real filesystem on +Linux; a real Windows/macOS filesystem was not exercised. +A failed lease acquisition preserves even an empty directory +and reports its created path, because another holder may be preparing it. + +The inline failure accumulator retains at most 32 rows within 2,048 +JSON-encoded UTF-8 bytes. It omits display names before omitting complete +address/code/reason rows. Recent output-I/O diagnostics retain at most five +rows within the same byte budget. The index retains all complete function +records. Per-item caveat collection retains at most eight first-observed +provider/text groups within an 8,000-byte JSON-encoded UTF-8 text budget, +instead of retaining all distinct caveats to rank them. +Omitted observations are counted without retaining their text; this count is +not a distinct-caveat count. Complete caveats remain in the owning index. + +Inventory and latest journal/index state still scale with the number of +functions and their retained metadata. Larger exports, long-running journal +histories and worst-case metadata expansion were not capacity-benchmarked; +this report makes no 100,000-function memory guarantee. The attempt limit is a +positive safe integer and does not allocate according to its numeric value. + +## Validation + +`npm run check` passed after the review fixes on `d3e0e9682`: type checking, +lint, formatting, dependency/metadata checks, module boundaries and architecture +guards. `npm run docs:check` also passed, including nine README conformance cases. After the main refresh, 226 tests passed across ten files, including the +four Ghidra profile suites and the six CLI/MCP/workflow/schema files. Earlier +Hopper lease and session-cleanup coverage below is additional; repeated tests +are not counted as new cases. + +- The focused integration run passed 112 tests across 11 files; four + macOS-specific real-Hopper startup cases were skipped on Linux. It includes + public CLI/MCP behavior, actual SDK schema conversion/dialect/client checks, + the shared lease extraction, session cleanup retry and descriptor ownership. +- After the final caveat/cleanup changes, 75 tests passed across the four + affected files: 48 export cases, 20 workflow-Evidence cases, six filesystem + ownership cases and one MCP caveat case. This adds five distinct cases to + the earlier integration coverage; overlapping executions are not summed. +- Follow-up review coverage passed 72 tests across the workflow and MCP + files, plus 22 filesystem/CLI/schema cases. This includes portable filename + reservations, conflicting legacy + journals, encoded caveat budgets and long failure names. Against the + published pre-fix implementation, all three new regression cases failed; + the complete SDK result for the long-name case was 485,937 bytes, above + the 51,200-byte tool window. The repaired case retains useful failures + inline and verifies complete names and caveats in the index. +- Original-PR controls reproduce the uppercase-input refusal, valid-journal + initialization cleanup race, lost failed-close ownership, unexpected-file + deletion, and the empty-root lease-acquisition race. These controls use the + original production implementation, not the repaired one. An additional + production selection control returns zero matching aliases before the fix + and the selected alias after it. A separate negative control of the + maintenance index-reuse change reproduced journal/index disagreement after + mutation of the append input; retaining the parsed snapshot resolves it. +- Three real Ghidra CLI calls passed: uppercase address range plus limit 1 + wrote one of two functions; resume with limit 100001 skipped the verified + file and wrote the other; an unmatched name selected zero and reported 298 + excluded inventory entries. Both output files matched the full MCP export. + The CLI group took 166.128 s elapsed, 120.848 s waited CPU and 1257.0 MiB + sampled peak family RSS, without a resource stop. + +Raw measurements and verifier scripts remain in the maintainer workspace at +`/tmp/rea-maintenance-20261023/6170/`, including completed and failed attempts, +full MCP exchanges, all function outputs and equivalence checks. They are local +verification artifacts, not shipped provider projects or committed binaries. +Real-provider coverage here is Linux x86-64 Ghidra only. Hopper, IDA, Windows and macOS +were not exercised with real providers by this verification. + +## CI attribution + +The first maintenance CI run used synthetic merge +`798091d99f5c9f434d832981e0c8d28a5ec4aabc` and exact base +`9fd78a74f84a8e924bd58e7deaf5934bfb24f53c`. All 54 observed failing +assertions reproduced on both revisions with identical failure diagnostics +after normalizing only their checkout paths: 11 source cases and 43 built +cases. Forty built failures came from cancelled-shard logs and were absent +from the aggregate report; cancelled coverage is not claimed as executed. + +The real-browser job also failed its stable-capture checkpoint. A bounded +checkpoint-only execution of the production verifier reproduced the same +`dom_structure: unchanged`, `scripts: unknown` result on exact base and +merge, with cleanup retained. Local Chromium was 155.0.8059.79 on Debian 12; +the CI Chrome version was not measured. This does not establish a passing +complete browser workflow. [The public CI attribution](https://github.com/morluto/rea/pull/6170#issuecomment-6373669748) +records the controls and separates these baseline failures from the three +actionable feature review findings fixed above. diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index e0a311bc44..717e65957f 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -126,6 +126,8 @@ export function createAnalysisExecution( /** Provider-neutral application capability for one analysis operation. */ export interface AnalysisOperationPort { + /** The session retains workflow resources whose final cleanup failed. */ + readonly resourceScope?: RetainedResourceScope | undefined; execute( operation: AnalysisOperation, parameters: Readonly>, diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 36966c5a7e..416546ad15 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -190,6 +190,7 @@ const ENHANCED_REQUIREMENTS: ReadonlyMap = new Map( get_objc_classes: ["list_names"], get_objc_protocols: ["list_names"], batch_decompile: ["procedure_pseudo_code"], + decompile_all: ["list_procedures", "procedure_pseudo_code"], get_call_graph: CALL_RELATIONSHIP_COMPOSITION, trace_call_path: CALL_RELATIONSHIP_COMPOSITION, analyze_swift_types: ["list_procedures"], diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 80c0cf9935..01e0cd8a5b 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -47,6 +47,7 @@ import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.j import { REA_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js"; import { workflowAnalysisProfile } from "./WorkflowAnalysisProfile.js"; import { + composedLimitationBudget, executeComposedWorkflow, createWorkflowEvidence, workflowSnapshotRecord, @@ -89,7 +90,8 @@ type DirectAnalysisTool = | "xrefs" | "trace_feature" | "trace_native_ui_action" - | "trace_native_values"; + | "trace_native_values" + | "decompile_all"; // These one-shot CLI questions require native utilities, not a deep engine. // Runtime observation retains the ordinary session and authority checks. @@ -463,7 +465,11 @@ const runAnalysis = async ( return err(recorded.error.retainPartialObservation(evidence)); } } - if (tool !== "trace_native_ui_action" && snapshotPath !== undefined) { + if ( + tool !== "trace_native_ui_action" && + tool !== "decompile_all" && + snapshotPath !== undefined + ) { const snapshot = session.exportAnalysisSnapshot(); if (!snapshot.ok) return err(snapshot.error.retainPartialObservation(evidence)); @@ -551,11 +557,15 @@ const executeAnalysisTool = async (input: { tool === "inspect_native_dispatch_metadata" || tool === "trace_feature" || tool === "trace_native_values" || - tool === "trace_native_ui_action" + tool === "trace_native_ui_action" || + tool === "decompile_all" ) { const { EnhancedTools } = await import("./EnhancedTools.js"); - const result = await executeComposedWorkflow(session, (observed) => - new EnhancedTools(observed).execute(tool, input.arguments, signal), + const result = await executeComposedWorkflow( + session, + (observed) => + new EnhancedTools(observed).execute(tool, input.arguments, signal), + composedLimitationBudget(tool), ); if (!result.ok) return result; const evidence = createWorkflowEvidence({ @@ -717,7 +727,8 @@ const allowsSnapshotReplay = ( | ManagedToolName | DirectAnalysisTool, ): boolean => { - if (tool === "trace_native_ui_action") return false; + if (tool === "trace_native_ui_action" || tool === "decompile_all") + return false; const descriptor = route.capabilities?.get(tool); return descriptor === undefined ? ![...(route.capabilities?.values() ?? [])].some( @@ -790,7 +801,8 @@ const isWorkflowEvidenceTool = ( tool === "inspect_native_dispatch_metadata" || tool === "trace_feature" || tool === "trace_native_values" || - tool === "trace_native_ui_action"; + tool === "trace_native_ui_action" || + tool === "decompile_all"; const fileExists = async (path: string): Promise => { try { diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index e00d2e83ed..dbc770e93b 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -37,6 +37,7 @@ import { import { inspectNativeDispatch } from "./native/NativeDispatchMetadataInspection.js"; import { traceNativeValues } from "./native/NativeValueTrace.js"; import { traceNativeUiAction } from "./native/NativeUiActionTrace.js"; +import { decompileAll } from "./decompileDirectory/decompileAll.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { addressedValueSchema } from "../domain/native/nativeInventory.js"; import { @@ -165,6 +166,12 @@ export class EnhancedTools { ? this.executeValidated({ name, input: parsed.data }, signal) : invalidEnhancedInput(name, parsed.error); } + case "decompile_all": { + const parsed = enhancedInputSchemas.decompile_all.safeParse(input); + return parsed.success + ? this.executeValidated({ name, input: parsed.data }, signal) + : invalidEnhancedInput(name, parsed.error); + } case "get_call_graph": { const parsed = enhancedInputSchemas.get_call_graph.safeParse(input); return parsed.success @@ -213,6 +220,8 @@ export class EnhancedTools { return this.#objcProtocols(signal); case "batch_decompile": return this.#batchDecompile(call.input.addresses, signal); + case "decompile_all": + return decompileAll(this.analysis, call.input, signal); case "get_call_graph": return this.#callGraph(call.input, signal); case "analyze_swift_types": diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index 333fa33d32..ee3629d433 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -213,6 +213,159 @@ it("deduplicates exact source caveats without merging providers or operations", ); }); +it("bounds per-item caveat retention while preserving inventory caveats", async () => { + const result = await executeComposedWorkflow( + { + execute: async (operation, parameters) => + ok( + createAnalysisExecution(null, provider, { + limitations: + operation === "list_procedures" + ? ["inventory caveat"] + : [ + "shared caveat", + `caveat at ${String(parameters.procedure)}`, + ], + }), + ), + }, + async (observed) => { + await observed.execute("list_procedures", {}); + for (const procedure of ["0x1", "0x2", "0x3"]) + await observed.execute("procedure_pseudo_code", { procedure }); + return ok({ count: 3 }); + }, + { + perItemOperations: new Set(["procedure_pseudo_code"]), + maxDistinct: 2, + retainedIn: "the files keep them", + }, + ); + expect(result).toEqual( + ok({ + result: { count: 3 }, + limitations: [ + "Fixture (fixture 1) list_procedures: inventory caveat", + "Fixture (fixture 1) procedure_pseudo_code: caveat at 0x1", + "Fixture (fixture 1) procedure_pseudo_code: shared caveat", + "2 per-item provider limitation observations were omitted from this answer; the files keep them.", + ], + }), + ); +}); + +it("spends one budget slot on a caveat that two per-item operations share", async () => { + const result = await executeComposedWorkflow( + { + execute: async (operation, parameters) => + ok( + createAnalysisExecution(null, provider, { + limitations: [ + "shared caveat", + `${operation} at ${String(parameters.procedure)}`, + ], + }), + ), + }, + async (observed) => { + await observed.execute("procedure_info", { procedure: "0x1" }); + await observed.execute("procedure_pseudo_code", { procedure: "0x1" }); + return ok(null); + }, + { + perItemOperations: new Set(["procedure_info", "procedure_pseudo_code"]), + maxDistinct: 2, + retainedIn: "the files keep them", + }, + ); + expect(result).toEqual( + ok({ + result: null, + limitations: [ + "Fixture (fixture 1) procedure_info, procedure_pseudo_code: shared caveat", + "Fixture (fixture 1) procedure_info: procedure_info at 0x1", + "1 per-item provider limitation observation was omitted from this answer; the files keep them.", + ], + }), + ); +}); + +it("counts repeated omitted observations without retaining their text for ranking", async () => { + const result = await executeComposedWorkflow( + { + execute: async (_operation, parameters) => + ok( + createAnalysisExecution(null, provider, { + limitations: [ + parameters.procedure === "first" + ? "first caveat" + : "later caveat", + ], + }), + ), + }, + async (observed) => { + await observed.execute("procedure_pseudo_code", { procedure: "first" }); + for (let index = 0; index < 30; index += 1) + await observed.execute("procedure_pseudo_code", { procedure: "later" }); + return ok(null); + }, + { + perItemOperations: new Set(["procedure_pseudo_code"]), + maxDistinct: 1, + retainedIn: "the index keeps them", + }, + ); + expect(result).toEqual( + ok({ + result: null, + limitations: [ + "Fixture (fixture 1) procedure_pseudo_code: first caveat", + "30 per-item provider limitation observations were omitted from this answer; the index keeps them.", + ], + }), + ); +}); + +it("bounds retained caveat text while preserving smaller later observations", async () => { + const result = await executeComposedWorkflow( + { + execute: async () => + ok( + createAnalysisExecution(null, provider, { + limitations: [ + "x".repeat(9_000), + "first", + "second", + "x".repeat(9_000), + "ok", + ], + }), + ), + }, + async (observed) => { + await observed.execute("procedure_pseudo_code", {}); + return ok(null); + }, + { + perItemOperations: new Set(["procedure_pseudo_code"]), + maxDistinct: 8, + maxBytes: 12, + retainedIn: "the index keeps them", + }, + ); + expect(result).toEqual( + ok({ + result: null, + limitations: [ + "Fixture (fixture 1) procedure_pseudo_code: first", + "Fixture (fixture 1) procedure_pseudo_code: ok", + "3 per-item provider limitation observations were omitted from this answer; the index keeps them.", + ], + }), + ); +}); + it("preserves a failed composed operation without an extra provider read", async () => { const failure = err(new AnalysisCancelledError("list_strings")); let calls = 0; diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index 25b1631fa2..1f105066d2 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -1,3 +1,4 @@ +import { jsonStringByteLength } from "../domain/jsonStringByteLength.js"; import { REA_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js"; import { workflowAnalysisProfile } from "./WorkflowAnalysisProfile.js"; import type { @@ -28,12 +29,45 @@ import { type SourceLimitation, } from "./SourceLimitations.js"; +/** + * Bound for caveats a workflow observes once per item. Provider caveats can + * name per-item addresses, so they do not deduplicate and would otherwise grow + * with the number of items. + */ +export interface ComposedLimitationBudget { + /** Operations the workflow calls once per item. */ + readonly perItemOperations: ReadonlySet; + /** + * First distinct per-item caveats retained, grouped by provider and text. + * Kept lines are rendered in the usual sorted order. + */ + readonly maxDistinct: number; + /** Retained caveat text as JSON-encoded UTF-8 bytes; defaults to 8,000. */ + readonly maxBytes?: number; + /** Where the omitted per-item caveats remain, as a clause. */ + readonly retainedIn: string; +} + +const DECOMPILE_ALL_LIMITATIONS: ComposedLimitationBudget = { + perItemOperations: new Set(["procedure_info", "procedure_pseudo_code"]), + maxDistinct: 8, + retainedIn: + "each function's record in decompile-index.json keeps the caveats of the observations it used", +}; + +/** The caveat budget for a composed workflow, when it has one. */ +export const composedLimitationBudget = ( + operation: string, +): ComposedLimitationBudget | undefined => + operation === "decompile_all" ? DECOMPILE_ALL_LIMITATIONS : undefined; + /** Retain the limits of exactly the observations used by one composed call. */ export const executeComposedWorkflow = async ( analysis: AnalysisOperationPort, execute: ( observed: AnalysisOperationPort, ) => Promise>, + budget?: ComposedLimitationBudget, ): Promise< Result< { @@ -45,8 +79,12 @@ export const executeComposedWorkflow = async ( > > => { const limitations = new Map(); + const perItemGroups = new Set(); + let perItemBytes = 0; + let omittedObservations = 0; let cachePolicy: "live" | undefined; const result = await execute({ + resourceScope: analysis.resourceScope, execute: async (operation, parameters, options) => { const execution = await analysis.execute(operation, parameters, options); if (execution.ok && execution.value.cachePolicy === "live") @@ -58,6 +96,33 @@ export const executeComposedWorkflow = async ( operation, limitation, } satisfies SourceLimitation; + if (budget?.perItemOperations.has(operation) === true) { + // Reject oversized text before allocating a grouping key. Complete + // caveats are already retained by the export's per-function owner. + const maxBytes = budget.maxBytes ?? 8_000; + const bytes = jsonStringByteLength(limitation, maxBytes); + if (bytes > maxBytes) { + omittedObservations += 1; + continue; + } + const group = JSON.stringify([ + fact.provider.id, + fact.provider.name, + fact.provider.version, + limitation, + ]); + if (!perItemGroups.has(group)) { + if ( + perItemGroups.size >= budget.maxDistinct || + perItemBytes + bytes > maxBytes + ) { + omittedObservations += 1; + continue; + } + perItemGroups.add(group); + perItemBytes += bytes; + } + } limitations.set(sourceLimitationKey(fact), fact); } return execution; @@ -66,7 +131,14 @@ export const executeComposedWorkflow = async ( if (!result.ok) return result; return ok({ result: result.value, - limitations: renderSourceLimitations([...limitations.values()]), + limitations: [ + ...renderSourceLimitations([...limitations.values()]), + ...(omittedObservations === 0 + ? [] + : [ + `${omittedObservations} per-item provider limitation observation${omittedObservations === 1 ? " was" : "s were"} omitted from this answer; ${budget?.retainedIn}.`, + ]), + ], ...(cachePolicy === undefined ? {} : { cachePolicy }), }); }; diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 340606c27b..bba3da1c5e 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -388,6 +388,7 @@ export class BinarySession /** External live state cannot be replayed by a direct or composed CLI workflow. */ allowsSnapshotReplay(operation: AnalysisOperation): boolean { + if (operation === "decompile_all") return false; if (this.#activeBundleResources !== undefined) return false; const capabilities = this.#active?.route.capabilities; const descriptor = capabilities?.get(operation); @@ -1141,6 +1142,7 @@ export class BinarySession if (!admission.ok) return admission; return ok( await operation({ + resourceScope: this.resourceScope, execute: (name, arguments_, options) => this.#execute(name, arguments_, options, Promise.resolve()), }), diff --git a/src/application/binary/BinarySessionRecords.ts b/src/application/binary/BinarySessionRecords.ts index 5942a164f1..7f68d2513c 100644 --- a/src/application/binary/BinarySessionRecords.ts +++ b/src/application/binary/BinarySessionRecords.ts @@ -28,6 +28,7 @@ import { err, ok, type Result } from "../../domain/result.js"; import type { AnalysisExecution, AnalysisOperation, + RetainedResourceScope, } from "../AnalysisProvider.js"; import { AnalysisSnapshotCache } from "./AnalysisSnapshotCache.js"; import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; @@ -70,6 +71,16 @@ export abstract class BinarySessionRecords { readonly #records: InvestigationRecords; protected readonly fileResources = new ArtifactResourceScope(); + readonly resourceScope: RetainedResourceScope = { + retain: (owner, resource) => + this.fileResources.retain({ + kind: "lease", + resource, + lease: { release: () => owner.close() }, + }), + run: (operation) => this.fileResources.run(operation), + }; + protected get hasPendingFileResources(): boolean { return this.fileResources.hasPending; } diff --git a/src/application/decompileDirectory/decompileAll.test.ts b/src/application/decompileDirectory/decompileAll.test.ts new file mode 100644 index 0000000000..4e7e3da2f6 --- /dev/null +++ b/src/application/decompileDirectory/decompileAll.test.ts @@ -0,0 +1,1706 @@ +import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +import { + createAnalysisExecution, + type AnalysisOperation, + type AnalysisOperationPort, +} from "../AnalysisProvider.js"; +import { EnhancedTools } from "../EnhancedTools.js"; +import { DecompileExport } from "./exportTree.js"; +import { + composedLimitationBudget, + createWorkflowEvidence, + executeComposedWorkflow, +} from "../WorkflowEvidence.js"; +import { toolContract } from "../../contracts/toolContracts.js"; +import { + DECOMPILE_DIRECTORY_INDEX_NAME, + DECOMPILE_DIRECTORY_JOURNAL_NAME, + decompileDirectoryIndexSchema, + decompileJournalLineSchema, +} from "../../contracts/decompileDirectory.js"; +import { createAnalysisProfile } from "../../domain/analysisProfile.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, +} from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok } from "../../domain/result.js"; + +const provider = { id: "fixture", name: "Fixture", version: "1" }; +const profile = createAnalysisProfile(provider, { decompiler: "fixture" }); +const otherProfile = createAnalysisProfile(provider, { decompiler: "other" }); +const targetSha = "ab".repeat(32); +const sentinel = "UNIQUE_PSEUDO_SENTINEL */"; + +interface Procedure { + readonly address: string; + readonly name: string; +} + +interface RecordedCall { + readonly operation: AnalysisOperation; + readonly procedure: unknown; + readonly origin: unknown; +} + +const subject = { + path: "/tmp/fixture", + sha256: targetSha, + format: "elf" as const, + architecture: "x86_64" as const, +}; + +const infoFor = ( + address: string, + ranges: readonly { readonly start: string; readonly end: string }[] = [ + { start: address, end: address }, + ], +) => { + const nonContiguous = ranges.length > 1; + const total = ranges.reduce((sum, range) => { + const start = BigInt(range.start); + const end = BigInt(range.end); + return sum + Number(end - start + 1n); + }, 0); + const span = nonContiguous + ? Number( + BigInt(ranges.at(-1)?.end ?? address) - + BigInt(ranges[0]?.start ?? address) + + 1n, + ) + : total; + return { + name: "fixture", + entrypoint: address, + basicblock_count: 1, + length: total, + signature: null, + locals: [], + classification: null, + body: { + available: true as const, + provenance: "fixture", + ranges, + total_bytes: total, + span_bytes: span, + non_contiguous: nonContiguous, + contains_entry: true, + }, + }; +}; + +/** procedure_info for an import: Ghidra reports an observed body with no ranges. */ +const externalInfoFor = (address: string) => ({ + ...infoFor(address), + basicblock_count: 0, + length: 0, + classification: { + external: true, + thunk: false, + thunk_target: null, + provenance: "fixture", + }, + body: { + available: true as const, + provenance: "fixture", + ranges: [], + total_bytes: 0, + span_bytes: 0, + non_contiguous: false, + contains_entry: false, + }, +}); + +const harness = (options?: { + readonly procedures?: readonly Procedure[]; + readonly fail?: Readonly< + Record + >; + readonly info?: "ok" | "unavailable" | "error"; + readonly subject?: boolean; + readonly mismatch?: string; + readonly unprofiled?: string; + readonly foreignTarget?: string; + /** Per-address caveats, as Ghidra reports terminal calls. */ + readonly addressCaveats?: boolean; + /** Addresses procedure_info classifies as external, as Ghidra does for imports. */ + readonly external?: readonly string[]; +}) => { + const procedures = options?.procedures ?? [ + { address: "0x3000", name: "Bar::late" }, + { address: "0x1000", name: "Foo::main" }, + { address: "0x2000", name: "Foo::helper" }, + ]; + const calls: RecordedCall[] = []; + const port: AnalysisOperationPort = { + execute: (operation, parameters, execution) => { + calls.push({ + operation, + procedure: parameters.procedure, + origin: execution?.selectorOrigin, + }); + if (operation === "list_procedures") + return Promise.resolve( + ok( + createAnalysisExecution( + procedures.map((procedure) => ({ + address: procedure.address, + value: procedure.name, + })), + provider, + { + analysisProfile: profile, + ...(options?.subject === false ? {} : { subject }), + }, + ), + ), + ); + if (operation === "procedure_info") { + if (options?.info === "unavailable") + return Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + provider.id, + operation, + "no body", + ), + ), + ); + if (options?.info === "error") + return Promise.resolve(err(new AnalysisInputError(operation))); + const address = String(parameters.procedure); + const info = + options?.external?.includes(address) === true + ? externalInfoFor(address) + : infoFor(address); + return Promise.resolve( + ok( + createAnalysisExecution(info, provider, { + analysisProfile: profile, + ...(options?.addressCaveats === true + ? { limitations: [`info caveat at ${address}`] } + : {}), + }), + ), + ); + } + if (operation === "procedure_pseudo_code") { + const address = String(parameters.procedure); + const failure = options?.fail?.[address]; + if (failure === "timeout") + return Promise.resolve(err(new AnalysisTimeoutError(operation, 1))); + if (failure === "error") + return Promise.resolve(err(new AnalysisInputError(operation))); + if (failure === "empty") + return Promise.resolve( + ok( + createAnalysisExecution("", provider, { + analysisProfile: profile, + }), + ), + ); + if (failure === "null") + return Promise.resolve( + ok( + createAnalysisExecution(null, provider, { + analysisProfile: profile, + }), + ), + ); + const selected = options?.mismatch === address ? otherProfile : profile; + return Promise.resolve( + ok( + createAnalysisExecution(`int fn(void) { ${sentinel} }`, provider, { + ...(options?.unprofiled === address + ? {} + : { analysisProfile: selected }), + ...(options?.foreignTarget === address + ? { subject: { ...subject, sha256: "cd".repeat(32) } } + : {}), + limitations: [ + "terminal call may cut the body", + ...(options?.addressCaveats === true + ? [`terminal call at ${address}`] + : []), + ], + }), + ), + ); + } + return Promise.resolve(err(new AnalysisInputError(operation))); + }, + }; + return { calls, port, tools: new EnhancedTools(port), procedures }; +}; + +const run = ( + tools: EnhancedTools, + directory: string, + input: Readonly> = {}, + signal?: AbortSignal, +) => + tools.execute( + "decompile_all", + { output_directory: directory, ...input }, + signal, + ); + +const of = (calls: readonly RecordedCall[], operation: AnalysisOperation) => + calls.filter((call) => call.operation === operation); + +const useOutputDirectory = (): (() => string) => { + let parent = ""; + beforeEach(async () => { + parent = await createTestTempDirectory("rea-decompile-all-"); + }); + return () => join(parent, "out"); +}; + +describe("decompile_all manifest", () => { + const directory = useOutputDirectory(); + + it("writes every function, keeps pseudocode out of the manifest, and records identity", async () => { + const { calls, tools } = harness(); + const result = await run(tools, directory()); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { + selected: 3, + excluded: 0, + requested: 3, + written: 3, + failed: 0, + skipped: 0, + remaining: 0, + }, + target_sha256: targetSha, + analysis_profile_digest: profile.digest, + provider, + layout: "address", + layout_status: "address", + failures: [], + }); + expect(JSON.stringify(result.value)).not.toContain(sentinel); + const text = await readFile( + join(directory(), "functions/0x1000.c"), + "utf8", + ); + expect( + text.startsWith( + "// rea decompile_all\n// name: Foo::main\n// address: 0x1000\n", + ), + ).toBe(true); + expect(text).toContain("// range: 0x1000-0x1000 inclusive"); + expect(text).toContain("// limitations: terminal call may cut the body"); + expect(text.slice(text.indexOf("\n\n") + 2)).toBe( + `int fn(void) { ${sentinel} }`, + ); + const index = await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ); + const journal = await readFile( + join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME), + "utf8", + ); + expect(index).not.toContain(sentinel); + expect(journal).not.toContain(sentinel); + expect( + decompileDirectoryIndexSchema.safeParse(JSON.parse(index)).success, + ).toBe(true); + expect( + of(calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x1000", "0x2000", "0x3000"]); + expect( + of(calls, "list_procedures").every((call) => call.origin === "provider"), + ).toBe(true); + const evidence = createWorkflowEvidence({ + target: undefined, + operation: "decompile_all", + parameters: { output_directory: directory() }, + result: result.value, + upstreamProfile: profile, + upstreamLimitations: [], + }); + expect(evidence.evidence_id.length).toBeGreaterThan(0); + expect( + toolContract("decompile_all").outputSchema.safeParse(evidence).success, + ).toBe(true); + expect(JSON.stringify(evidence)).not.toContain(sentinel); + }); + + it("records one decompiler failure without aborting or writing that file", async () => { + const { tools } = harness({ + fail: { "0x2000": "error", "0x3000": "timeout" }, + }); + const result = await run(tools, directory()); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { written: 1, failed: 2, requested: 3 }, + }); + const failures = ( + result.value as { failures: { address: string; code: string }[] } + ).failures; + expect(failures.map((failure) => failure.code).sort()).toEqual( + ["deadline", "invalid_request"].sort(), + ); + await expect( + readFile(join(directory(), "functions/0x2000.c")), + ).rejects.toThrow(); + await expect( + readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).resolves.toContain(sentinel); + }); +}); + +describe("decompile_all external functions", () => { + const directory = useOutputDirectory(); + + it("reports an external function as external without requesting its pseudocode", async () => { + const { calls, tools } = harness({ external: ["0x2000"] }); + const result = await run(tools, directory()); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { requested: 3, written: 2, failed: 0, external: 1 }, + failures: [], + }); + expect( + of(calls, "procedure_pseudo_code").map((call) => call.procedure), + ).not.toContain("0x2000"); + await expect( + readFile(join(directory(), "functions/0x2000.c")), + ).rejects.toThrow(); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect(index.functions.find((entry) => entry.address === "0x2000")).toEqual( + { + address: "0x2000", + name: "Foo::helper", + status: "external", + body: { available: true, ranges: [], non_contiguous: false }, + }, + ); + }); + + it("skips a recorded external function on resume without asking the provider again", async () => { + const { calls, tools } = harness({ external: ["0x1000", "0x2000"] }); + expect((await run(tools, directory())).ok).toBe(true); + calls.length = 0; + const resumed = await run(tools, directory(), { resume: true, limit: 1 }); + + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + // A limit below the external count still finishes: nothing remains. + expect(resumed.value).toMatchObject({ + counts: { requested: 0, skipped: 3, remaining: 0, external: 0 }, + }); + expect(of(calls, "procedure_info")).toEqual([]); + expect(of(calls, "procedure_pseudo_code")).toEqual([]); + }); + + it("reports a conflicting file at a written function's path when a resume finds it external", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + await writeFile(join(directory(), "functions/0x2000.c"), "replaced\n"); + + const resumed = await run( + harness({ external: ["0x2000"] }).tools, + directory(), + { resume: true }, + ); + + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { requested: 1, skipped: 2, failed: 1, external: 0 }, + failures: [{ address: "0x2000", code: "output_exists" }], + }); + }); + + it("resumes an import recorded as invalid pseudocode by an earlier version as external", async () => { + const before = harness({ fail: { "0x2000": "null" } }); + expect((await run(before.tools, directory())).ok).toBe(true); + const journal = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const text = await readFile(journal, "utf8"); + expect(text).toContain('"no_pseudocode"'); + // The journal an earlier version wrote for the same import. + await writeFile( + journal, + text.replace( + /"code":"no_pseudocode","message":"[^"]*"/u, + '"code":"invalid_pseudocode","message":"The provider returned pseudocode that is not text."', + ), + ); + + const after = harness({ external: ["0x2000"] }); + const resumed = await run(after.tools, directory(), { resume: true }); + + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { requested: 1, skipped: 2, failed: 0, external: 1 }, + failures: [], + }); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect( + index.functions.find((entry) => entry.address === "0x2000")?.status, + ).toBe("external"); + }); + + it.each([ + ["a failure", '"failure":{"code":"x","message":"x"},'], + ["a file path", '"relative_path":"functions/0x2000.c",'], + ["a checksum", `"sha256":"${"cd".repeat(32)}",`], + ])( + "refuses to resume a journal whose external record carries %s", + async (_, field) => { + const { tools } = harness({ external: ["0x2000"] }); + expect((await run(tools, directory())).ok).toBe(true); + const journal = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const text = await readFile(journal, "utf8"); + await writeFile( + journal, + text.replace('"status":"external",', `"status":"external",${field}`), + ); + + const resumed = await run(tools, directory(), { resume: true }); + + expect(resumed.ok).toBe(false); + if (resumed.ok) return; + expect(resumed.error).toBeInstanceOf(AnalysisInputError); + if (resumed.error instanceof AnalysisInputError) + expect(resumed.error.userMessage).toContain("not a compatible export"); + }, + ); + + it("reports a null pseudocode result as no pseudocode, not as text of the wrong type", async () => { + const { tools } = harness({ fail: { "0x2000": "null" } }); + const result = await run(tools, directory()); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { written: 2, failed: 1, external: 0 }, + failures: [{ address: "0x2000", code: "no_pseudocode" }], + }); + }); +}); + +describe("decompile_all resume", () => { + const directory = useOutputDirectory(); + + it("refuses an existing directory until resume matches, then skips verified files", async () => { + const first = harness(); + const created = await run(first.tools, directory()); + expect(created.ok).toBe(true); + const again = harness(); + const refused = await run(again.tools, directory()); + expect(refused.ok).toBe(false); + if (refused.ok) return; + expect(refused.error).toBeInstanceOf(AnalysisInputError); + expect(refused.error.userMessage).toContain("already exists"); + expect(of(again.calls, "list_procedures")).toHaveLength(0); + + const resumed = harness(); + const second = await run(resumed.tools, directory(), { resume: true }); + expect(second.ok).toBe(true); + if (!second.ok) return; + expect(second.value).toMatchObject({ + counts: { written: 0, skipped: 3, failed: 0, requested: 0 }, + resume: true, + }); + expect(of(resumed.calls, "procedure_pseudo_code")).toHaveLength(0); + }); + + it("does not rewrite a conflicting file and retries a missing one", async () => { + const { tools } = harness(); + expect((await run(tools, directory())).ok).toBe(true); + const file = join(directory(), "functions/0x1000.c"); + await writeFile(file, "tampered"); + await rm(join(directory(), "functions/0x2000.c")); + const next = harness(); + const resumed = await run(next.tools, directory(), { resume: true }); + + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { written: 1, failed: 1, skipped: 1 }, + }); + expect(await readFile(file, "utf8")).toBe("tampered"); + expect( + of(next.calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x2000"]); + const failures = ( + resumed.value as { failures: { address: string; code: string }[] } + ).failures; + expect(failures).toEqual([ + expect.objectContaining({ address: "0x1000", code: "output_exists" }), + ]); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect( + index.functions.find((record) => record.address === "0x1000")?.body, + ).toEqual({ + available: true, + ranges: [{ start: "0x1000", end: "0x1000" }], + non_contiguous: false, + }); + }); + + it("refuses resume for a different target digest or a damaged journal", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + const journalPath = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const original = await readFile(journalPath, "utf8"); + const damaged = original.replace(targetSha, "cd".repeat(32)); + await writeFile(journalPath, damaged); + const mismatched = await run(harness().tools, directory(), { + resume: true, + }); + expect(mismatched.ok).toBe(false); + if (!mismatched.ok) + expect(mismatched.error.userMessage).toContain("does not match"); + + await writeFile(journalPath, `${original}not-json\n`); + const broken = await run(harness().tools, directory(), { resume: true }); + expect(broken.ok).toBe(false); + if (!broken.ok) + expect(broken.error.userMessage).toContain("not a compatible export"); + expect( + await readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).toContain(sentinel); + }); +}); + +describe("decompile_all filters and layout", () => { + const directory = useOutputDirectory(); + + it("filters by name, namespace, and address and reports the excluded count", async () => { + const { tools } = harness(); + const result = await run(tools, directory(), { + name: "helper", + namespace: "Foo::", + address_start: "0x1500", + address_end: "0x2500", + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { selected: 1, excluded: 2, written: 1 }, + }); + expect( + await readFile(join(directory(), "functions/0x2000.c"), "utf8"), + ).toContain("Foo::helper"); + }); + + it("stops at a limit and continues in address order on resume", async () => { + const first = harness(); + const limited = await run(first.tools, directory(), { limit: 1 }); + expect(limited.ok).toBe(true); + if (!limited.ok) return; + expect(limited.value).toMatchObject({ + counts: { written: 1, remaining: 2, skipped: 0 }, + }); + expect( + of(first.calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x1000"]); + + const second = harness(); + const resumed = await run(second.tools, directory(), { + resume: true, + limit: 1, + }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { written: 1, skipped: 1, remaining: 1 }, + }); + expect( + of(second.calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x2000"]); + }); + + it("reports dwarf layout unavailable and still uses address file names", async () => { + const { tools } = harness(); + const result = await run(tools, directory(), { layout: "dwarf" }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + layout: "dwarf", + layout_status: "debug_layout_unavailable", + }); + const limitations = ( + result.value as { limitations: string[] } + ).limitations.join("\n"); + expect(limitations).toContain("unavailable"); + expect( + await readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).toContain(sentinel); + const names = await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ); + expect(names).not.toContain("Foo/"); + }); + + it("keeps a non-contiguous procedure_info body in the file header", async () => { + const procedures = [{ address: "0x1000", name: "main" }]; + const calls: RecordedCall[] = []; + const port: AnalysisOperationPort = { + execute: (operation, parameters) => { + calls.push({ + operation, + procedure: parameters.procedure, + origin: undefined, + }); + if (operation === "list_procedures") + return Promise.resolve( + ok( + createAnalysisExecution( + procedures.map((procedure) => ({ + address: procedure.address, + value: procedure.name, + })), + provider, + { analysisProfile: profile, subject }, + ), + ), + ); + if (operation === "procedure_info") + return Promise.resolve( + ok( + createAnalysisExecution( + infoFor("0x1000", [ + { start: "0x1000", end: "0x1007" }, + { start: "0x1010", end: "0x1017" }, + ]), + provider, + { analysisProfile: profile }, + ), + ), + ); + return Promise.resolve( + ok( + createAnalysisExecution( + `int main(void) { ${sentinel} }`, + provider, + { analysisProfile: profile }, + ), + ), + ); + }, + }; + const result = await new EnhancedTools(port).execute("decompile_all", { + output_directory: directory(), + }); + expect(result.ok).toBe(true); + const text = await readFile( + join(directory(), "functions/0x1000.c"), + "utf8", + ); + expect(text).toContain("// range: 0x1000-0x1007 inclusive"); + expect(text).toContain("// range: 0x1010-0x1017 inclusive"); + expect(text).toContain("// non_contiguous: true"); + }); + + it("still writes when procedure_info is unavailable and does not ask again", async () => { + const { calls, tools } = harness({ info: "unavailable" }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ counts: { written: 3, failed: 0 } }); + expect( + (result.value as { limitations: string[] }).limitations.join("\n"), + ).toContain("procedure_info"); + expect(of(calls, "procedure_info")).toHaveLength(1); + expect( + await readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).toContain("// body: unavailable"); + }); +}); + +describe("decompile_all partial results", () => { + const directory = useOutputDirectory(); + + it("caps inline failures and keeps the rest in the index", async () => { + const procedures = Array.from({ length: 33 }, (_, index) => ({ + address: `0x${(index + 1).toString(16)}`, + name: `fn${index.toString(16)}`, + })); + const { tools } = harness({ + procedures, + fail: Object.fromEntries( + procedures.map((procedure) => [procedure.address, "error" as const]), + ), + }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { failed: 33, written: 0 }, + }); + const shown = (result.value as { failures: unknown[] }).failures; + expect(shown.length).toBeGreaterThan(0); + expect(shown.length).toBeLessThanOrEqual(32); + expect(result.value).toMatchObject({ failures_omitted: 33 - shown.length }); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect( + index.functions.filter((record) => record.status === "failed"), + ).toHaveLength(33); + expect(JSON.stringify(index)).not.toContain(sentinel); + }); + + it("leaves committed files in place when the caller cancels", async () => { + const controller = new AbortController(); + const procedures = [ + { address: "0x1000", name: "first" }, + { address: "0x2000", name: "second" }, + ]; + const port: AnalysisOperationPort = { + execute: (operation, parameters) => { + if ( + operation === "procedure_pseudo_code" && + parameters.procedure === "0x2000" + ) + controller.abort(); + if (operation === "list_procedures") + return Promise.resolve( + ok( + createAnalysisExecution( + procedures.map((procedure) => ({ + address: procedure.address, + value: procedure.name, + })), + provider, + { analysisProfile: profile, subject }, + ), + ), + ); + if (operation === "procedure_info") + return Promise.resolve( + ok( + createAnalysisExecution( + infoFor(String(parameters.procedure)), + provider, + { analysisProfile: profile }, + ), + ), + ); + return Promise.resolve( + ok( + createAnalysisExecution( + "int cancelled(void) { return 1; }", + provider, + { analysisProfile: profile, limitations: [] }, + ), + ), + ); + }, + }; + const result = await new EnhancedTools(port).execute( + "decompile_all", + { output_directory: directory() }, + controller.signal, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + await expect( + readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).resolves.toContain("int cancelled(void)"); + await expect( + readFile(join(directory(), "functions/0x2000.c")), + ).rejects.toThrow(); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect(index.functions.map((record) => record.address)).toEqual(["0x1000"]); + const next = harness({ procedures }); + const resumed = await run(next.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ counts: { written: 1, skipped: 1 } }); + expect( + of(next.calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x2000"]); + }); + + it("rejects an inverted address range before creating a directory or calling a provider", async () => { + const { calls, tools } = harness(); + const result = await run(tools, directory(), { + address_start: "ram:0x20", + address_end: "ram:0x10", + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.userMessage).toContain("before address start"); + expect(calls).toHaveLength(0); + await expect(readFile(directory())).rejects.toThrow(); + }); +}); + +describe("decompile_all caller coordinates", () => { + const directory = useOutputDirectory(); + it("accepts uppercase caller coordinates and retains address-space identity", async () => { + const { tools } = harness({ + info: "unavailable", + procedures: [ + { address: "RAM%3Aoverlay:0xabcd", name: "in_range" }, + { address: "ram%3Aoverlay:0xabcd", name: "other_space" }, + { address: "RAM%3Aoverlay:0xabce", name: "past_end" }, + ], + }); + const result = await run(tools, directory(), { + address_start: "RAM%3aoverlay:0X00ABCD", + address_end: "RAM%3Aoverlay:0xABCD", + }); + expect(result.ok).toBe(true); + if (result.ok) + expect(result.value).toMatchObject({ + counts: { selected: 1, excluded: 2, written: 1 }, + }); + }); +}); + +describe("decompile_all duplicate-address labels", () => { + const directory = useOutputDirectory(); + it("checks later labels before coalescing a selected address", async () => { + const { tools } = harness({ + procedures: [ + { address: "0x1000", name: "Other::original" }, + { address: "0x1000", name: "Chosen::alias" }, + { address: "0x1000", name: "Chosen::alias_again" }, + { address: "0x2000", name: "Other::helper" }, + ], + }); + const result = await run(tools, directory(), { + name: "alias", + namespace: "Chosen::", + }); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { selected: 1, excluded: 1, written: 1 }, + }); + expect( + await readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).toContain("// name: Chosen::alias\n"); + }); +}); + +describe("decompile_all portable function paths", () => { + const directory = useOutputDirectory(); + const procedures = [ + { address: "RAM%3Aoverlay:0xabcd", name: "upper_space" }, + { address: "ram%3Aoverlay:0xabcd", name: "lower_space" }, + ]; + it("exports distinct case-sensitive address spaces using case-insensitive-unique filenames", async () => { + const { tools } = harness({ procedures, info: "unavailable" }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { selected: 2, written: 2, failed: 0 }, + }); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + const names = index.functions.map((record) => + record.relative_path!.toLowerCase(), + ); + expect(new Set(names).size).toBe(2); + expect(index.functions.map((record) => record.address)).toEqual( + procedures.map((row) => row.address), + ); + const resumed = await run(tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + if (resumed.ok) + expect(resumed.value).toMatchObject({ + counts: { skipped: 2, written: 0, failed: 0 }, + }); + }); + it("refuses legacy journal paths that alias under portable filename rules", async () => { + const { tools } = harness({ procedures, info: "unavailable" }); + expect((await run(tools, directory())).ok).toBe(true); + const journalPath = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const lines = (await readFile(journalPath, "utf8")) + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)); + lines[2].relative_path = lines[1].relative_path.toLowerCase(); + await writeFile( + journalPath, + lines.map((line) => JSON.stringify(line)).join("\n") + "\n", + ); + const refused = await run(tools, directory(), { resume: true }); + expect(refused.ok).toBe(false); + if (!refused.ok) + expect(refused.error.userMessage).toContain("compatible export"); + }); +}); + +describe("decompile_all identity and recovery", () => { + const directory = useOutputDirectory(); + + it("records an empty target digest and refuses to resume an unknown target", async () => { + const { tools } = harness({ subject: false }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ target_sha256: null }); + expect( + (result.value as { limitations: string[] }).limitations.join("\n"), + ).toContain("SHA-256"); + const other = harness({ subject: false }); + const resumed = await run(other.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(false); + if (!resumed.ok) + expect(resumed.error.userMessage).toContain("target SHA-256"); + expect(of(other.calls, "procedure_pseudo_code")).toHaveLength(0); + }); + + it("does not accept a decompiler observation without the export's profile or for another target", async () => { + const { tools } = harness({ + unprofiled: "0x2000", + foreignTarget: "0x3000", + }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ counts: { written: 1, failed: 2 } }); + expect((result.value as { failures: unknown[] }).failures).toEqual([ + expect.objectContaining({ address: "0x2000", code: "profile_changed" }), + expect.objectContaining({ address: "0x3000", code: "target_changed" }), + ]); + }); + + it("refuses to resume an export whose analysis profile is unknown", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + const journalPath = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const original = await readFile(journalPath, "utf8"); + await writeFile( + journalPath, + original.replace( + `"analysis_profile_digest":"${profile.digest}"`, + `"analysis_profile_digest":null`, + ), + ); + const next = harness(); + const resumed = await run(next.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(false); + if (!resumed.ok) + expect(resumed.error.userMessage).toContain("analysis profile digest"); + expect(of(next.calls, "procedure_pseudo_code")).toHaveLength(0); + }); + + it("refuses resume when the output directory does not exist", async () => { + const { calls, tools } = harness(); + const result = await run(tools, directory(), { resume: true }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.userMessage).toContain("existing export directory"); + expect(calls).toHaveLength(0); + await expect( + readFile(join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME)), + ).rejects.toThrow(); + }); + + it("exports addressed rows and counts inventory rows without an address", async () => { + const { tools } = harness({ + procedures: [ + { address: "", name: "ghost" }, + { address: "0x1000", name: "main" }, + ], + }); + const result = await run(tools, directory()); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.value).toMatchObject({ + counts: { selected: 1, written: 1, failed: 0 }, + }); + expect( + (result.value as { limitations: string[] }).limitations.join("\n"), + ).toContain("1 inventory row(s) had no address"); + }); + + it("resumes after a torn final journal line and keeps the journal parseable", async () => { + const first = harness(); + expect((await run(first.tools, directory(), { limit: 1 })).ok).toBe(true); + const journalPath = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + await writeFile( + journalPath, + `${await readFile(journalPath, "utf8")}{"kind":"function","name":"é\u00e9`, + ); + const stray = join(directory(), ".tmp-0123456789abcdef.c"); + await writeFile(stray, "not owned by this run"); + const resumed = await run(harness().tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { written: 2, skipped: 1, failed: 0 }, + }); + const text = await readFile(journalPath, "utf8"); + expect(text.endsWith("\n")).toBe(true); + for (const line of text.trimEnd().split("\n")) + expect( + decompileJournalLineSchema.safeParse(JSON.parse(line)).success, + ).toBe(true); + // A file is not this export's just because its name looks like a temporary. + await expect(readFile(stray, "utf8")).resolves.toBe( + "not owned by this run", + ); + }); +}); + +describe("decompile_all untrusted directory and provider facts", () => { + const directory = useOutputDirectory(); + + it("refuses a journal that is a symlink and leaves its target unchanged", async () => { + expect((await run(harness().tools, directory(), { limit: 1 })).ok).toBe( + true, + ); + const journalPath = join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME); + const outside = join(directory(), "..", "outside.jsonl"); + const torn = `${await readFile(journalPath, "utf8")}{"kind":"func`; + await writeFile(outside, torn); + await rm(journalPath); + await symlink(outside, journalPath); + const next = harness(); + const resumed = await run(next.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(false); + expect(of(next.calls, "procedure_pseudo_code")).toHaveLength(0); + expect(await readFile(outside, "utf8")).toBe(torn); + }); + + it("does not take body ranges from procedure_info for another entry point", async () => { + const port: AnalysisOperationPort = { + execute: (operation) => { + if (operation === "list_procedures") + return Promise.resolve( + ok( + createAnalysisExecution( + [{ address: "0x1000", value: "main" }], + provider, + { analysisProfile: profile, subject }, + ), + ), + ); + if (operation === "procedure_info") + return Promise.resolve( + ok( + createAnalysisExecution(infoFor("0x9000"), provider, { + analysisProfile: profile, + }), + ), + ); + return Promise.resolve( + ok( + createAnalysisExecution( + `int main(void) { ${sentinel} }`, + provider, + { + analysisProfile: profile, + }, + ), + ), + ); + }, + }; + const result = await new EnhancedTools(port).execute("decompile_all", { + output_directory: directory(), + }); + expect(result.ok).toBe(true); + const text = await readFile( + join(directory(), "functions/0x1000.c"), + "utf8", + ); + expect(text).toContain("// body: unavailable"); + expect(text).not.toContain("0x9000"); + }); + + it("keeps a function's file name after a failed retry that recorded no path", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + await rm(join(directory(), "functions/0x1000.c")); + const failing = harness({ fail: { "0x1000": "timeout" } }); + expect((await run(failing.tools, directory(), { resume: true })).ok).toBe( + true, + ); + const retried = await run(harness().tools, directory(), { resume: true }); + expect(retried.ok).toBe(true); + if (!retried.ok) return; + expect(retried.value).toMatchObject({ counts: { written: 1, skipped: 2 } }); + await expect( + readFile(join(directory(), "functions/0x1000.c"), "utf8"), + ).resolves.toContain(sentinel); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect( + index.functions.find((record) => record.address === "0x1000") + ?.relative_path, + ).toBe("functions/0x1000.c"); + }); + + it("tries functions not yet attempted before retrying recorded failures", async () => { + const failing = { fail: { "0x1000": "timeout" as const } }; + expect( + (await run(harness(failing).tools, directory(), { limit: 1 })).ok, + ).toBe(true); + const next = harness(failing); + const resumed = await run(next.tools, directory(), { + resume: true, + limit: 1, + }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect( + of(next.calls, "procedure_pseudo_code").map((call) => call.procedure), + ).toEqual(["0x2000"]); + expect(resumed.value).toMatchObject({ + counts: { written: 1, failed: 0, remaining: 2 }, + }); + }); +}); + +describe("decompile_all journal recovery", () => { + const directory = useOutputDirectory(); + + const indexRecord = async (address: string) => + decompileDirectoryIndexSchema + .parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ) + .functions.find((record) => record.address === address); + + it("keeps a written file's checksum through a read error and skips it once readable", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + const file = join(directory(), "functions/0x1000.c"); + const original = await readFile(file); + const written = await indexRecord("0x1000"); + // A directory in the file's place makes the read fail without being absent. + await rm(file); + await mkdir(file); + const blocked = await run(harness().tools, directory(), { resume: true }); + expect(blocked.ok).toBe(true); + if (!blocked.ok) return; + expect(blocked.value).toMatchObject({ + counts: { failed: 1, skipped: 2, written: 0 }, + failures: [ + expect.objectContaining({ address: "0x1000", code: "output_io" }), + ], + }); + expect((await indexRecord("0x1000"))?.status).toBe("failed"); + + await rm(file, { recursive: true }); + await writeFile(file, original); + const next = harness(); + const recovered = await run(next.tools, directory(), { resume: true }); + expect(recovered.ok).toBe(true); + if (!recovered.ok) return; + expect(recovered.value).toMatchObject({ + counts: { failed: 0, skipped: 3, written: 0 }, + }); + expect(of(next.calls, "procedure_pseudo_code")).toHaveLength(0); + expect(await indexRecord("0x1000")).toEqual(written); + }); + + it("bounds per-function provider caveats in the answer and keeps each in its file", async () => { + const procedures = Array.from({ length: 20 }, (_, index) => ({ + address: `0x${(0x1000 + index * 0x10).toString(16)}`, + name: `fn_${index}`, + })); + const { port } = harness({ procedures, addressCaveats: true }); + const composed = await executeComposedWorkflow( + port, + (observed) => + new EnhancedTools(observed).execute("decompile_all", { + output_directory: directory(), + }), + composedLimitationBudget("decompile_all"), + ); + expect(composed.ok).toBe(true); + if (!composed.ok) return; + const { limitations } = composed.value; + // The shared caveat is kept; 39 per-address caveats collapse to a count. + expect(limitations).toHaveLength(9); + expect(limitations).toContainEqual( + expect.stringContaining("terminal call may cut the body"), + ); + expect(limitations.at(-1)).toBe( + "33 per-item provider limitation observations were omitted from this answer; each function's record in decompile-index.json keeps the caveats of the observations it used.", + ); + const last = await readFile( + join(directory(), "functions/0x1130.c"), + "utf8", + ); + expect(last).toContain("info caveat at 0x1130"); + expect(last).toContain("terminal call at 0x1130"); + }); + + it("keeps the caveats each attempt used in its index record", async () => { + const { tools } = harness({ + addressCaveats: true, + fail: { "0x1000": "timeout", "0x2000": "empty" }, + }); + expect((await run(tools, directory())).ok).toBe(true); + expect((await indexRecord("0x1000"))?.limitations).toEqual([ + "info caveat at 0x1000", + ]); + // The fixture's empty pseudocode observation reports no caveats. + expect((await indexRecord("0x2000"))?.limitations).toEqual([ + "info caveat at 0x2000", + ]); + expect((await indexRecord("0x3000"))?.limitations).toEqual([ + "info caveat at 0x3000", + "terminal call may cut the body", + "terminal call at 0x3000", + ]); + }); + + it("keeps caveats beyond the header's length cap in the index", async () => { + const caveats = Array.from( + { length: 40 }, + (_, index) => + `terminal call at 0x${(0x1000 + index).toString(16)} ${"x".repeat(150)}`, + ); + const port: AnalysisOperationPort = { + execute: (operation) => + Promise.resolve( + ok( + operation === "list_procedures" + ? createAnalysisExecution( + [{ address: "0x1000", value: "large" }], + provider, + { analysisProfile: profile, subject }, + ) + : operation === "procedure_info" + ? createAnalysisExecution(infoFor("0x1000"), provider, { + analysisProfile: profile, + }) + : createAnalysisExecution( + `int large(void) { ${sentinel} }`, + provider, + { + analysisProfile: profile, + limitations: caveats, + }, + ), + ), + ), + }; + const result = await new EnhancedTools(port).execute("decompile_all", { + output_directory: directory(), + }); + expect(result.ok).toBe(true); + const header = await readFile( + join(directory(), "functions/0x1000.c"), + "utf8", + ); + expect(header).toContain("// limitations_truncated: true"); + expect(header).not.toContain(caveats.at(-1)); + expect((await indexRecord("0x1000"))?.limitations).toEqual(caveats); + }); +}); + +describe("decompile_all late cancellation", () => { + const directory = useOutputDirectory(); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("reports cancellation when the caller aborts while the index is written", async () => { + const controller = new AbortController(); + const reason = new Error("caller stopped"); + const writeIndex = DecompileExport.prototype.writeIndex; + vi.spyOn(DecompileExport.prototype, "writeIndex").mockImplementation( + function (this: DecompileExport, input) { + controller.abort(reason); + return writeIndex.call(this, input); + }, + ); + const result = await run( + harness().tools, + directory(), + {}, + controller.signal, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + // Finalization still completed, so the export resumes without new work. + vi.restoreAllMocks(); + const next = harness(); + const resumed = await run(next.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + expect(of(next.calls, "procedure_pseudo_code")).toHaveLength(0); + }); + + it("keeps an index failure primary when the caller also aborts", async () => { + const controller = new AbortController(); + const failure = new AnalysisOutputError("decompile_all", "index failed"); + vi.spyOn(DecompileExport.prototype, "writeIndex").mockImplementation(() => { + controller.abort(new Error("caller stopped")); + return Promise.resolve(err(failure)); + }); + const result = await run( + harness().tools, + directory(), + {}, + controller.signal, + ); + expect(result).toEqual(err(failure)); + }); +}); + +describe("decompile_all cancelled publish loop", () => { + const directory = useOutputDirectory(); + afterEach(() => { + vi.restoreAllMocks(); + }); + + const stopReason = new Error("caller stopped"); + + /** A port that aborts the caller's signal once the first function is decompiled. */ + const abortAfterFirst = (controller: AbortController) => { + const { port } = harness(); + const stopping: AnalysisOperationPort = { + execute: async (operation, parameters, execution) => { + const result = await port.execute(operation, parameters, execution); + if (operation === "procedure_pseudo_code") controller.abort(stopReason); + return result; + }, + }; + return new EnhancedTools(stopping); + }; + + it("keeps an index failure primary when cancellation stops the publish loop", async () => { + const controller = new AbortController(); + const failure = new AnalysisOutputError("decompile_all", "index failed"); + vi.spyOn(DecompileExport.prototype, "writeIndex").mockResolvedValue( + err(failure), + ); + const result = await run( + abortAfterFirst(controller), + directory(), + {}, + controller.signal, + ); + expect(result).toEqual(err(failure)); + }); + + it("keeps a journal close failure primary when cancellation stops the publish loop", async () => { + const controller = new AbortController(); + const failure = new AnalysisOutputError("decompile_all", "close failed"); + const close = DecompileExport.prototype.close; + vi.spyOn(DecompileExport.prototype, "close").mockImplementation( + async function (this: DecompileExport) { + await close.call(this); + return err(failure); + }, + ); + const result = await run( + abortAfterFirst(controller), + directory(), + {}, + controller.signal, + ); + expect(result).toEqual(err(failure)); + }); + + it("reports the caller's reason when cancellation stops the publish loop and finalization succeeds", async () => { + const controller = new AbortController(); + const result = await run( + abortAfterFirst(controller), + directory(), + {}, + controller.signal, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(stopReason); + }); + + it("reports the caller's reason when procedure_info returns its own cancellation as the caller aborts", async () => { + const controller = new AbortController(); + const { port } = harness(); + const cancelling: AnalysisOperationPort = { + execute: (operation, parameters, execution) => { + if (operation !== "procedure_info") + return port.execute(operation, parameters, execution); + controller.abort(stopReason); + return Promise.resolve( + err( + new AnalysisCancelledError("procedure_info", { + cause: new Error("provider session stopped"), + }), + ), + ); + }, + }; + const result = await run( + new EnhancedTools(cancelling), + directory(), + {}, + controller.signal, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(stopReason); + }); + + it("keeps a provider's own cancellation primary when the caller did not abort", async () => { + const { port } = harness(); + const providerCancelled = new AnalysisCancelledError( + "procedure_pseudo_code", + { cause: new Error("provider session stopped") }, + ); + const cancelling: AnalysisOperationPort = { + execute: (operation, parameters, execution) => + operation === "procedure_pseudo_code" + ? Promise.resolve(err(providerCancelled)) + : port.execute(operation, parameters, execution), + }; + vi.spyOn(DecompileExport.prototype, "writeIndex").mockResolvedValue( + err(new AnalysisOutputError("decompile_all", "index failed")), + ); + const result = await run( + new EnhancedTools(cancelling), + directory(), + {}, + new AbortController().signal, + ); + expect(result).toEqual(err(providerCancelled)); + }); +}); + +describe("decompile_all output directory lease", () => { + const directory = useOutputDirectory(); + + /** A port that holds the first decompile until the test lets it finish. */ + const heldAtFirstDecompile = () => { + const { port } = harness(); + let release = (): void => undefined; + const released = new Promise((resolve) => { + release = resolve; + }); + let reached = (): void => undefined; + const reachedFirst = new Promise((resolve) => { + reached = resolve; + }); + let held = false; + const holding: AnalysisOperationPort = { + execute: async (operation, parameters, execution) => { + if (operation === "procedure_pseudo_code" && !held) { + held = true; + reached(); + await released; + } + return port.execute(operation, parameters, execution); + }, + }; + return { tools: new EnhancedTools(holding), reachedFirst, release }; + }; + + it("refuses a second run on a directory another run is exporting, then resumes after it", async () => { + const first = heldAtFirstDecompile(); + const exporting = run(first.tools, directory()); + await first.reachedFirst; + + const competing = harness(); + const refused = await run(competing.tools, directory(), { resume: true }); + expect(refused.ok).toBe(false); + if (refused.ok) return; + // Busy is retryable once the holder finishes, unlike invalid input. + expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + expect(refused.error).toMatchObject({ + resource: "capacity", + retryable: true, + reportedLimits: { holder_process_id: process.pid }, + }); + // The refused run neither decompiled nor appended to the journal. + expect(of(competing.calls, "procedure_pseudo_code")).toHaveLength(0); + + first.release(); + const exported = await exporting; + expect(exported.ok).toBe(true); + const journal = ( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_JOURNAL_NAME), + "utf8", + ) + ) + .trimEnd() + .split("\n"); + expect(journal).toHaveLength(4); + + // The lease ends with the run, and the next run sees every record. + const next = harness(); + const resumed = await run(next.tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ counts: { skipped: 3, written: 0 } }); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect(index.functions.map((record) => record.status)).toEqual([ + "written", + "written", + "written", + ]); + }); + + it("releases the lease when a resume is refused before any work", async () => { + expect((await run(harness().tools, directory())).ok).toBe(true); + const foreign = harness({ subject: false }); + const refused = await run(foreign.tools, directory(), { resume: true }); + expect(refused.ok).toBe(false); + const resumed = await run(harness().tools, directory(), { resume: true }); + expect(resumed.ok).toBe(true); + }); +}); + +describe("decompile_all repeated output failures", () => { + const directory = useOutputDirectory(); + + const procedures = Array.from({ length: 8 }, (_, index) => ({ + address: `0x${(0x1000 + index * 0x10).toString(16)}`, + name: `${"long-name".repeat(400)}_${index.toString()}`, + })); + + /** Put a directory where each listed function's file was, so reading it fails. */ + const blockFiles = async (addresses: readonly string[]) => { + for (const address of addresses) { + const file = join(directory(), `functions/${address}.c`); + await rm(file); + await mkdir(file); + } + }; + + it("stops after five consecutive output_io failures and keeps what it recorded", async () => { + expect((await run(harness({ procedures }).tools, directory())).ok).toBe( + true, + ); + await blockFiles(procedures.slice(0, 7).map(({ address }) => address)); + const next = harness({ procedures }); + const stopped = await run(next.tools, directory(), { resume: true }); + expect(stopped.ok).toBe(false); + if (stopped.ok) return; + expect(stopped.error).toBeInstanceOf(AnalysisOutputError); + expect(stopped.error.userMessage).toContain( + "Stopped after 5 function files in a row could not be written or read", + ); + expect(stopped.error.diagnostics).toMatchObject({ + attempted: 5, + failed: 5, + consecutive_output_io: 5, + }); + expect( + Buffer.byteLength(JSON.stringify(stopped.error.diagnostics)), + ).toBeLessThan(4_096); + // The index was still written and records the five failures; the sixth + // and seventh blocked functions were not attempted. + const index = decompileDirectoryIndexSchema.parse( + JSON.parse( + await readFile( + join(directory(), DECOMPILE_DIRECTORY_INDEX_NAME), + "utf8", + ), + ), + ); + expect( + index.functions + .filter((record) => record.failure?.code === "output_io") + .map((record) => record.address), + ).toEqual(procedures.slice(0, 5).map(({ address }) => address)); + expect( + index.functions.find( + (record) => record.address === procedures[5]?.address, + )?.status, + ).toBe("written"); + }); + + it("does not stop when a verified file interrupts the failures", async () => { + expect((await run(harness({ procedures }).tools, directory())).ok).toBe( + true, + ); + // Four failures, one verified file, then three more failures. + await blockFiles( + [0, 1, 2, 3, 5, 6, 7].map((index) => procedures[index]?.address ?? ""), + ); + const resumed = await run(harness({ procedures }).tools, directory(), { + resume: true, + }); + expect(resumed.ok).toBe(true); + if (!resumed.ok) return; + expect(resumed.value).toMatchObject({ + counts: { failed: 7, skipped: 1, remaining: 0 }, + }); + }); +}); diff --git a/src/application/decompileDirectory/decompileAll.ts b/src/application/decompileDirectory/decompileAll.ts new file mode 100644 index 0000000000..4bf91ba4b2 --- /dev/null +++ b/src/application/decompileDirectory/decompileAll.ts @@ -0,0 +1,994 @@ +import { join } from "node:path"; +import { z } from "zod"; + +import { + DECOMPILE_DIRECTORY_INDEX_NAME, + DECOMPILE_DIRECTORY_INLINE_FAILURE_LIMIT, + DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES, + decompileDirectoryManifestSchema, + type DecompileDirectoryFunctionRecord, + type DecompileDirectoryInput, + type DecompileDirectoryInlineFailure, +} from "../../contracts/decompileDirectory.js"; +import { jsonValueByteLength } from "../../domain/jsonValueByteLength.js"; +import { digestSchema } from "../../domain/digests.js"; +import type { + AnalysisExecution, + AnalysisOperationPort, +} from "../AnalysisProvider.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisOutputError, + AnalysisTimeoutError, +} from "../../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { procedureInfoSchema } from "../../domain/native/procedureInfo.js"; +import { addressedValueSchema } from "../../domain/native/nativeInventory.js"; +import { addressCoordinate } from "../../domain/native/addressCoordinate.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + classifyOutputPath, + DecompileExport, + type ExportIdentity, +} from "./exportTree.js"; +import { + bodyRecord, + functionFileText, + type FunctionBody, +} from "./functionFile.js"; +import { + selectProcedures, + validateDecompileBounds, + type SelectedProcedure, +} from "./selection.js"; + +const DWARF_LAYOUT_UNAVAILABLE = + "Debug compilation-unit layout is unavailable. Files are named by canonical address."; +const BODY_UNAVAILABLE = + "procedure_info is unavailable, so function body ranges are unknown."; +const TARGET_DIGEST_UNAVAILABLE = + "Target SHA-256 was not observed, so this export cannot be resumed."; +const PROFILE_UNAVAILABLE = + "The provider did not commit an analysis profile, so this export cannot be resumed."; +const DUPLICATE_ADDRESSES = + "Duplicate procedure addresses were coalesced to the first matching inventory row."; +/** Consecutive output_io failures after which an export stops. */ +export const OUTPUT_IO_STREAK_LIMIT = 5; +const unaddressedRows = (count: number): string => + `${count.toString()} inventory row(s) had no address and were not exported.`; + +interface FailureReason { + readonly code: string; + readonly message: string; +} + +type InlineFailure = DecompileDirectoryInlineFailure; +type NamedFailure = Extract; + +interface PublishState { + /** Functions this call took on after skipping verified files; bounded by limit. */ + attempted: number; + written: number; + failed: number; + /** Functions procedure_info classifies as external: not decompiled, no file. */ + external: number; + skipped: number; + readonly failures: InlineFailure[]; + inlineFailureBytes: number; + readonly recentFailures: PublishState["failures"]; + bodyUnavailable: boolean; + /** Failure code of the function being attempted, if it failed. */ + lastFailureCode: string | undefined; + /** + * Consecutive output_io failures. A verified file, a write, or any other + * outcome ends the run of failures; a recorded external function does not. + */ + outputIoStreak: number; +} + +/** + * Write one pseudocode file per selected function and return only a manifest. + * Files already committed stay in place when a later function fails or the + * caller cancels. + */ +export const decompileAll = async ( + analysis: AnalysisOperationPort, + input: DecompileDirectoryInput, + signal?: AbortSignal, +): Promise> => { + const bounds = validateDecompileBounds(input); + if (!bounds.ok) return bounds; + const placement = await classifyOutputPath( + input.output_directory, + input.resume, + ); + if (!placement.ok) return placement; + if (isAborted(signal)) return err(cancelled(signal)); + const loaded = await loadProcedures(analysis, input.layout, signal); + if (!loaded.ok) return loaded; + if (isAborted(signal)) return err(cancelled(signal)); + const selection = selectProcedures( + loaded.value.procedures, + input, + bounds.value, + ); + const opened = await DecompileExport.open({ + placement: placement.value, + identity: loaded.value.identity, + resourceScope: analysis.resourceScope, + }); + if (!opened.ok) return opened; + const session = opened.value; + const state: PublishState = { + attempted: 0, + written: 0, + failed: 0, + external: 0, + skipped: 0, + failures: [], + inlineFailureBytes: 2, + recentFailures: [], + bodyUnavailable: false, + lastFailureCode: undefined, + outputIoStreak: 0, + }; + let published: Result | undefined; + let indexed: Result | undefined; + let limitations: readonly string[] = []; + let closed: Result | undefined; + // The journal handle is closed on every path, including an unexpected throw. + try { + published = await publishSelected({ + analysis, + session, + procedures: attemptOrder(session, selection.procedures), + identity: loaded.value.identity, + signal, + limit: input.limit, + state, + }); + limitations = exportLimitations( + input.layout, + loaded.value.identity, + selection.collapsedDuplicates, + state.bodyUnavailable, + loaded.value.unaddressed, + ); + indexed = await session.writeIndex({ + layoutStatus: layoutStatus(input.layout), + limitations, + }); + } finally { + closed = await session.close(); + } + // A publish, index or close failure stays primary: the journal or index + // may be incomplete and the caller needs that reason. Cancellation only + // replaces success, so an index or close failure after the caller's abort + // stopped the publish loop is still reported. A provider's own + // cancellation, without a caller abort, is a publish failure like any other. + const callerStoppedPublish = + !published.ok && + published.error instanceof AnalysisCancelledError && + isAborted(signal); + const failures = [ + ...(!published.ok && !callerStoppedPublish ? [published.error] : []), + ...(indexed.ok ? [] : [indexed.error]), + ...(closed.ok ? [] : [closed.error]), + ]; + const primary = failures[0]; + if (primary !== undefined) { + for (const failure of failures.slice(1)) + primary.retainSecondaryFailure(failure); + return !closed.ok && + primary !== closed.error && + closed.error.cleanupIncomplete + ? err( + analysisErrorWithCleanupFailure( + primary, + closed.error, + "decompile_all", + ).bindCleanupDependencies([closed.error]), + ) + : err(primary); + } + // Finalization succeeded, so committed files and the index are usable on + // resume, but a caller abort that stopped the loop or arrived meanwhile is + // not a success. Report it with the caller's own reason. + if (isAborted(signal)) return err(cancelled(signal)); + const manifest = decompileDirectoryManifestSchema.safeParse( + manifestOf({ + root: session.root, + input, + identity: loaded.value.identity, + selected: selection.procedures.length, + excluded: selection.excluded, + state, + limitations, + }), + ); + return manifest.success + ? ok(manifest.data) + : err( + new AnalysisOutputError( + "decompile_all", + "decompile manifest could not be built", + ), + ); +}; + +/** + * Functions with no recorded failure go first, so a limited resume reaches + * new work instead of retrying the same failures every call. + */ +const attemptOrder = ( + session: DecompileExport, + procedures: readonly SelectedProcedure[], +): readonly SelectedProcedure[] => { + const failedBefore = (procedure: SelectedProcedure): boolean => + session.record(procedure.address)?.status === "failed"; + return [ + ...procedures.filter((procedure) => !failedBefore(procedure)), + ...procedures.filter(failedBefore), + ]; +}; + +const loadProcedures = async ( + analysis: AnalysisOperationPort, + layout: DecompileDirectoryInput["layout"], + signal: AbortSignal | undefined, +): Promise< + Result< + { + readonly procedures: readonly SelectedProcedure[]; + readonly identity: ExportIdentity; + readonly unaddressed: number; + }, + AnalysisError + > +> => { + const observed = await analysis.execute( + "list_procedures", + {}, + executionOptions(signal), + ); + if (!observed.ok) return observed; + const parsed = z.array(addressedValueSchema).safeParse(observed.value.result); + if (!parsed.success) + return err( + new AnalysisOutputError( + "list_procedures", + "provider returned an invalid procedure inventory", + ), + ); + const identity = identityFrom(observed.value, layout); + if (!identity.ok) return identity; + // An empty address cannot name a file or a journal record. + const addressed = parsed.data.filter((row) => row.address.length > 0); + return ok({ + procedures: addressed.map((row) => ({ + address: row.address, + name: row.value.length === 0 ? null : row.value, + })), + identity: identity.value, + unaddressed: parsed.data.length - addressed.length, + }); +}; + +const identityFrom = ( + execution: AnalysisExecution, + layout: DecompileDirectoryInput["layout"], +): Result => { + const sha = execution.subject?.sha256 ?? null; + if (sha !== null && !digestSchema.safeParse(sha).success) + return err( + new AnalysisOutputError( + "list_procedures", + "provider returned a target digest that is not a SHA-256", + ), + ); + const profile = execution.analysisProfile?.digest ?? null; + if (profile !== null && !digestSchema.safeParse(profile).success) + return err( + new AnalysisOutputError( + "list_procedures", + "provider returned an analysis profile digest that is not a SHA-256", + ), + ); + const provider = execution.provider; + if (provider.id.length === 0 || provider.name.length === 0) + return err( + new AnalysisOutputError( + "list_procedures", + "provider identity is incomplete", + ), + ); + if (provider.version !== null && provider.version.length === 0) + return err( + new AnalysisOutputError( + "list_procedures", + "provider identity is incomplete", + ), + ); + return ok({ + targetSha256: sha, + provider: { + id: provider.id, + name: provider.name, + version: provider.version, + }, + analysisProfileDigest: profile, + layout, + }); +}; + +interface PublishInput { + readonly analysis: AnalysisOperationPort; + readonly session: DecompileExport; + readonly procedures: readonly SelectedProcedure[]; + readonly identity: ExportIdentity; + readonly signal: AbortSignal | undefined; + readonly limit: number | undefined; + readonly state: PublishState; +} + +const publishSelected = async ( + input: PublishInput, +): Promise> => { + let bodySupport: "unknown" | "unavailable" = "unknown"; + for (const procedure of input.procedures) { + if (isAborted(input.signal)) return err(cancelled(input.signal)); + const existing = await existingFile( + input.session, + procedure.address, + input.signal, + ); + if (isAborted(input.signal)) return err(cancelled(input.signal)); + if (existing === "skip") { + input.state.skipped += 1; + // A file that reads back and verifies shows the directory works. + input.state.outputIoStreak = 0; + // The file verified again after a later failure, so the journal and + // index report it as written instead of keeping the stale failure. + const written = input.session.lastWritten(procedure.address); + if ( + written !== undefined && + input.session.record(procedure.address) !== written + ) { + const appended = await input.session.append(written); + if (!appended.ok) return appended; + } + continue; + } + // The journal header pins the target, provider and profile, so a + // recorded external classification still holds. Asking again would only + // spend the limit, and a limit below the external count would leave + // functions remaining on every resume. + // A conflicting or unreadable file at its reserved path is still reported + // below rather than hidden by the skip. + if ( + existing === "write" && + input.session.record(procedure.address)?.status === "external" + ) { + input.state.skipped += 1; + continue; + } + if (input.limit !== undefined && input.state.attempted >= input.limit) + continue; + input.state.attempted += 1; + input.state.lastFailureCode = undefined; + const outcome = await attemptOne(input, procedure, existing, bodySupport); + bodySupport = outcome.bodySupport; + if (!outcome.result.ok) return outcome.result; + input.state.outputIoStreak = + input.state.lastFailureCode === "output_io" + ? input.state.outputIoStreak + 1 + : 0; + // Repeated write failures mean the directory has stopped accepting files, + // for example a full disk. Stop instead of decompiling every remaining + // function only to fail its write. Committed files and records are kept. + if (input.state.outputIoStreak >= OUTPUT_IO_STREAK_LIMIT) + return err(outputStoppedAccepting(input.state)); + } + return ok(undefined); +}; + +/** One function this call takes on: report a file problem or decompile it. */ +const attemptOne = async ( + input: PublishInput, + procedure: SelectedProcedure, + existing: "write" | "conflict" | "io", + bodySupport: "unknown" | "unavailable", +): Promise<{ + readonly result: Result; + readonly bodySupport: "unknown" | "unavailable"; +}> => { + if (existing === "conflict") { + const reason = { + code: "output_exists", + message: + "The function file already exists and does not match this export.", + }; + recordFailure(input.state, procedure, reason); + const appended = await input.session.append( + failureRecord( + procedure, + retainedBody(input.session, procedure.address), + reason, + { relativePath: input.session.pathFor(procedure.address) }, + ), + ); + return { result: appended, bodySupport }; + } + if (existing === "io") { + const reason = { + code: "output_io", + message: "The existing function file could not be read.", + }; + recordFailure(input.state, procedure, reason); + const appended = await input.session.append( + failureRecord( + procedure, + retainedBody(input.session, procedure.address), + reason, + { relativePath: input.session.pathFor(procedure.address) }, + ), + ); + return { result: appended, bodySupport }; + } + return publishOne({ + ...input, + procedure, + bodySupport, + }); +}; + +const publishOne = async (input: { + readonly analysis: AnalysisOperationPort; + readonly session: DecompileExport; + readonly procedure: SelectedProcedure; + readonly identity: ExportIdentity; + readonly signal: AbortSignal | undefined; + readonly state: PublishState; + readonly bodySupport: "unknown" | "unavailable"; +}): Promise<{ + readonly result: Result; + readonly bodySupport: "unknown" | "unavailable"; +}> => { + const body = await readBody( + input.analysis, + input.procedure.address, + input.identity, + input.signal, + input.bodySupport, + ); + if (body.kind === "cancelled") + return { result: err(body.error), bodySupport: input.bodySupport }; + if (body.kind === "unavailable") input.state.bodyUnavailable = true; + // Caveats of the observations this attempt used. The journal record keeps + // the complete list, written or failed, so the bounded Evidence summary and + // the length-capped file header never hold the only copy. + const infoCaveats = body.kind === "body" ? (body.limitations ?? []) : []; + // procedure_info classifies the function as external, so this target holds + // no code of its own to decompile. The record keeps the provider's body + // fact as observed; it is reported as external, not as a failure. + if (body.kind === "body" && body.external === true) { + input.state.external += 1; + return { + result: await input.session.append( + externalRecord(input.procedure, body.fact, infoCaveats), + ), + bodySupport: body.support, + }; + } + const decompiled = await input.analysis.execute( + "procedure_pseudo_code", + { procedure: input.procedure.address }, + executionOptions(input.signal), + ); + if (isAborted(input.signal)) + return { result: err(cancelled(input.signal)), bodySupport: body.support }; + if (!decompiled.ok) { + if (decompiled.error instanceof AnalysisCancelledError) + return { result: decompiled, bodySupport: body.support }; + const reason = failureReason(decompiled.error); + recordFailure(input.state, input.procedure, reason); + return { + result: await input.session.append( + failureRecord(input.procedure, body.fact, reason, { + limitations: infoCaveats, + }), + ), + bodySupport: body.support, + }; + } + const mismatch = observationMismatch(input.identity, decompiled.value); + if (mismatch !== undefined) { + const reason = mismatch; + recordFailure(input.state, input.procedure, reason); + return { + result: await input.session.append( + failureRecord(input.procedure, body.fact, reason, { + limitations: infoCaveats, + }), + ), + bodySupport: body.support, + }; + } + const caveats = [ + ...new Set([...infoCaveats, ...decompiled.value.limitations]), + ]; + const pseudocode = decompiled.value.result; + if (typeof pseudocode !== "string" || pseudocode.length === 0) { + const reason = pseudocodeFailure(pseudocode); + recordFailure(input.state, input.procedure, reason); + return { + result: await input.session.append( + failureRecord(input.procedure, body.fact, reason, { + limitations: caveats, + }), + ), + bodySupport: body.support, + }; + } + const relativePath = input.session.pathFor(input.procedure.address); + const bytes = Buffer.from( + functionFileText({ + procedure: input.procedure, + body: body.fact, + identity: input.identity, + limitations: caveats, + pseudocode, + }), + "utf8", + ); + const committed = await input.session.commitFunctionFile(relativePath, bytes); + if (committed.status === "conflict" || committed.status === "io") { + const reason = + committed.status === "conflict" + ? { + code: "output_exists", + message: + "The function file already exists and does not match this export.", + } + : { + code: "output_io", + message: "The function file could not be written.", + }; + recordFailure(input.state, input.procedure, reason); + return { + result: await input.session.append( + failureRecord(input.procedure, body.fact, reason, { + relativePath, + limitations: caveats, + }), + ), + bodySupport: body.support, + }; + } + const appended = await input.session.append({ + address: input.procedure.address, + name: input.procedure.name, + status: "written", + relative_path: relativePath, + sha256: committed.sha256, + body: bodyRecord(body.fact), + // The header may truncate; the journal keeps the complete list. + ...(caveats.length === 0 ? {} : { limitations: caveats }), + }); + if (!appended.ok) return { result: appended, bodySupport: body.support }; + input.state.written += committed.status === "written" ? 1 : 0; + input.state.skipped += committed.status === "matched" ? 1 : 0; + return { result: ok(undefined), bodySupport: body.support }; +}; + +const readBody = async ( + analysis: AnalysisOperationPort, + address: string, + identity: ExportIdentity, + signal: AbortSignal | undefined, + support: "unknown" | "unavailable", +): Promise< + | { + readonly kind: "body"; + readonly fact: FunctionBody; + readonly support: "unknown" | "unavailable"; + /** + * Caveats of a procedure_info observation for this export's target and + * profile. An observation for another target or profile is foreign and + * contributes none. + */ + readonly limitations?: readonly string[]; + /** + * procedure_info for this export classifies the function as external. + * Absent when the classification is unknown or not external. + */ + readonly external?: true; + } + | { + readonly kind: "unavailable"; + readonly fact: FunctionBody; + readonly support: "unavailable"; + } + | { readonly kind: "cancelled"; readonly error: AnalysisCancelledError } +> => { + if (support === "unavailable") + return { + kind: "unavailable", + fact: unavailableBody(BODY_UNAVAILABLE), + support, + }; + const observed = await analysis.execute( + "procedure_info", + { procedure: address }, + executionOptions(signal), + ); + if (!observed.ok) { + if (observed.error instanceof AnalysisCancelledError) + return { kind: "cancelled", error: observed.error }; + if (observed.error instanceof AnalysisCapabilityUnavailableError) + return { + kind: "unavailable", + fact: unavailableBody(BODY_UNAVAILABLE), + support: "unavailable", + }; + return { + kind: "body", + fact: unavailableBody(failureReason(observed.error).message), + support, + }; + } + const mismatch = observationMismatch(identity, observed.value); + if (mismatch !== undefined) + return { + kind: "body", + fact: unavailableBody(`procedure_info: ${mismatch.message}`), + support, + }; + const parsed = procedureInfoSchema.safeParse(observed.value.result); + if (!parsed.success) + return { + kind: "body", + fact: unavailableBody("procedure_info returned an invalid body."), + support, + limitations: observed.value.limitations, + }; + if (!sameAddress(parsed.data.entrypoint, address)) + return { + kind: "body", + fact: unavailableBody( + "procedure_info described a different entry point than this function.", + ), + support, + limitations: observed.value.limitations, + }; + const external = + parsed.data.classification?.external === true + ? { external: true as const } + : {}; + if (!parsed.data.body.available) + return { + kind: "body", + fact: unavailableBody(parsed.data.body.reason), + support, + limitations: observed.value.limitations, + ...external, + }; + return { + kind: "body", + fact: { + available: true, + ranges: parsed.data.body.ranges, + non_contiguous: parsed.data.body.non_contiguous, + }, + support, + limitations: observed.value.limitations, + ...external, + }; +}; + +type ExistingFile = "skip" | "write" | "conflict" | "io"; + +const existingFile = async ( + session: DecompileExport, + address: string, + signal: AbortSignal | undefined, +): Promise => { + const record = session.lastWritten(address); + if ( + record === undefined || + record.sha256 === undefined || + record.relative_path === undefined + ) + return "write"; + const digest = await session + .hashFile(record.relative_path, signal) + .catch((cause: unknown) => cause); + if (typeof digest === "string") + return digest === record.sha256 ? "skip" : "conflict"; + return nodeAbsent(digest) ? "write" : "io"; +}; + +const PROFILE_CHANGED: FailureReason = { + code: "profile_changed", + message: + "The decompiler observation does not match this export's provider or analysis profile.", +}; + +const TARGET_CHANGED: FailureReason = { + code: "target_changed", + message: + "The decompiler observation names a different target than this export.", +}; + +/** Why a per-function observation cannot join this export, if it cannot. */ +const observationMismatch = ( + identity: ExportIdentity, + execution: AnalysisExecution, +): FailureReason | undefined => { + const provider = execution.provider; + if ( + provider.id !== identity.provider.id || + provider.name !== identity.provider.name || + provider.version !== identity.provider.version + ) + return PROFILE_CHANGED; + // A missing profile is unknown, not a match for a committed one. + if ( + (execution.analysisProfile?.digest ?? null) !== + identity.analysisProfileDigest + ) + return PROFILE_CHANGED; + // A per-function observation may omit its subject; one that names a + // different target is not this export's target. + const target = execution.subject?.sha256; + return target === undefined || target === identity.targetSha256 + ? undefined + : TARGET_CHANGED; +}; + +/** Same canonical address; numeric when both parse, otherwise exact text. */ +const sameAddress = (left: string, right: string): boolean => { + const a = addressCoordinate(left); + const b = addressCoordinate(right); + return a === null || b === null + ? left === right + : a.space === b.space && a.offset === b.offset; +}; + +const failureReason = (error: AnalysisError): FailureReason => { + if (error instanceof AnalysisTimeoutError) + return { code: "deadline", message: "The decompiler deadline elapsed." }; + const projected = projectAnalysisError(error); + const message = projected.message.replace(/\s+/gu, " ").trim().slice(0, 300); + return { + code: projected.code.slice(0, 80), + message: message.length === 0 ? projected.code : message, + }; +}; + +const outputStoppedAccepting = (state: PublishState): AnalysisOutputError => + new AnalysisOutputError( + "decompile_all", + "output directory stopped accepting function files", + { + userMessage: `Stopped after ${OUTPUT_IO_STREAK_LIMIT.toString()} function files in a row could not be written or read, for example because the disk is full or permissions changed. Files already written are kept and recorded in ${DECOMPILE_DIRECTORY_INDEX_NAME}. Fix the output directory, then resume.`, + remediationAction: + "Free space or restore write access in the output directory, then rerun with resume.", + diagnostics: { + attempted: state.attempted, + written: state.written, + failed: state.failed, + external: state.external, + skipped: state.skipped, + consecutive_output_io: state.outputIoStreak, + last_failures: state.recentFailures.map((failure) => ({ ...failure })), + last_failures_omitted: + Math.min(state.failed, OUTPUT_IO_STREAK_LIMIT) - + state.recentFailures.length, + }, + }, + ); + +const recordFailure = ( + state: PublishState, + procedure: SelectedProcedure, + reason: FailureReason, +): void => { + state.failed += 1; + state.lastFailureCode = reason.code; + const failure = { + address: procedure.address, + name: procedure.name, + code: reason.code, + message: reason.message, + }; + if (state.failures.length < DECOMPILE_DIRECTORY_INLINE_FAILURE_LIMIT) { + const comma = state.failures.length === 0 ? 0 : 1; + const fitted = fitInlineFailure( + failure, + DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES - + state.inlineFailureBytes - + comma, + ); + if (fitted !== undefined) { + state.failures.push(fitted.row); + state.inlineFailureBytes += comma + fitted.bytes; + } + } + const recent = fitInlineFailure( + failure, + DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES - 2, + ); + // An oversized latest row cannot leave an older example labelled as recent. + if (recent === undefined) state.recentFailures.length = 0; + else state.recentFailures.push(recent.row); + while ( + state.recentFailures.length > OUTPUT_IO_STREAK_LIMIT || + jsonValueByteLength( + state.recentFailures, + DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES, + ) > DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES + ) + state.recentFailures.shift(); +}; + +const fitInlineFailure = ( + failure: NamedFailure, + budget: number, +): { readonly row: InlineFailure; readonly bytes: number } | undefined => { + const bytes = jsonValueByteLength(failure, budget); + if (bytes <= budget) return { row: failure, bytes }; + // Name is display context; address remains the complete lookup identity. + const { name: _name, ...withoutName } = failure; + const compactBytes = jsonValueByteLength(withoutName, budget); + return compactBytes <= budget + ? { row: withoutName, bytes: compactBytes } + : undefined; +}; + +const failureRecord = ( + procedure: SelectedProcedure, + body: FunctionBody, + reason: FailureReason, + options: { + readonly relativePath?: string; + /** Caveats of the observations this attempt used. */ + readonly limitations?: readonly string[]; + } = {}, +): DecompileDirectoryFunctionRecord => ({ + address: procedure.address, + name: procedure.name, + status: "failed", + body: bodyRecord(body), + failure: reason, + ...(options.relativePath === undefined + ? {} + : { relative_path: options.relativePath }), + ...(options.limitations === undefined || options.limitations.length === 0 + ? {} + : { limitations: [...options.limitations] }), +}); + +/** Why a successful pseudocode observation still produced no file. */ +const pseudocodeFailure = (pseudocode: unknown): FailureReason => { + if (pseudocode === null) + return { + code: "no_pseudocode", + message: "The provider returned null pseudocode for this function.", + }; + return typeof pseudocode === "string" + ? { + code: "empty_pseudocode", + message: "The provider returned no pseudocode.", + } + : { + code: "invalid_pseudocode", + message: "The provider returned pseudocode that is not text.", + }; +}; + +const externalRecord = ( + procedure: SelectedProcedure, + body: FunctionBody, + limitations: readonly string[], +): DecompileDirectoryFunctionRecord => ({ + address: procedure.address, + name: procedure.name, + status: "external", + body: bodyRecord(body), + ...(limitations.length === 0 ? {} : { limitations: [...limitations] }), +}); + +const unavailableBody = (reason: string): FunctionBody => ({ + available: false, + reason: reason.length === 0 ? "unavailable" : reason, +}); + +/** Keep the journal's body fact when a conflicting file is left unchanged. */ +const retainedBody = ( + session: DecompileExport, + address: string, +): FunctionBody => { + return session.record(address)?.body ?? unavailableBody("not rewritten"); +}; + +const exportLimitations = ( + layout: DecompileDirectoryInput["layout"], + identity: ExportIdentity, + collapsedDuplicates: boolean, + bodyUnavailable: boolean, + unaddressed: number, +): readonly string[] => [ + ...(layout === "dwarf" ? [DWARF_LAYOUT_UNAVAILABLE] : []), + ...(identity.targetSha256 === null ? [TARGET_DIGEST_UNAVAILABLE] : []), + ...(identity.analysisProfileDigest === null ? [PROFILE_UNAVAILABLE] : []), + ...(collapsedDuplicates ? [DUPLICATE_ADDRESSES] : []), + ...(unaddressed > 0 ? [unaddressedRows(unaddressed)] : []), + ...(bodyUnavailable ? [BODY_UNAVAILABLE] : []), +]; + +const layoutStatus = ( + layout: DecompileDirectoryInput["layout"], +): "address" | "debug_layout_unavailable" => + layout === "dwarf" ? "debug_layout_unavailable" : "address"; + +const manifestOf = (input: { + readonly root: string; + readonly input: DecompileDirectoryInput; + readonly identity: ExportIdentity; + readonly selected: number; + readonly excluded: number; + readonly state: PublishState; + readonly limitations: readonly string[]; +}) => { + const requested = + input.state.written + input.state.failed + input.state.external; + const shown = input.state.failures.slice( + 0, + DECOMPILE_DIRECTORY_INLINE_FAILURE_LIMIT, + ); + return { + output_directory: input.root, + index_path: join(input.root, DECOMPILE_DIRECTORY_INDEX_NAME), + layout: input.input.layout, + layout_status: layoutStatus(input.input.layout), + counts: { + selected: input.selected, + excluded: input.excluded, + requested, + written: input.state.written, + failed: input.state.failed, + external: input.state.external, + skipped: input.state.skipped, + remaining: input.selected - input.state.skipped - requested, + }, + failures: shown, + failures_omitted: input.state.failed - shown.length, + provider: input.identity.provider, + analysis_profile_digest: input.identity.analysisProfileDigest, + target_sha256: input.identity.targetSha256, + limitations: [...input.limitations], + resume: input.input.resume, + }; +}; + +const executionOptions = (signal: AbortSignal | undefined) => ({ + ...(signal === undefined ? {} : { signal }), + selectorOrigin: "provider" as const, +}); + +const cancelled = (signal: AbortSignal | undefined): AnalysisCancelledError => + new AnalysisCancelledError("decompile_all", { cause: signal?.reason }); + +const isAborted = (signal: AbortSignal | undefined): boolean => + signal?.aborted === true; + +const nodeAbsent = (cause: unknown): boolean => + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === "ENOENT"; diff --git a/src/application/decompileDirectory/exportResources.ts b/src/application/decompileDirectory/exportResources.ts new file mode 100644 index 0000000000..2d83f38548 --- /dev/null +++ b/src/application/decompileDirectory/exportResources.ts @@ -0,0 +1,262 @@ +import { + link, + lstat, + mkdtemp, + open, + rename, + rmdir, + unlink, + type FileHandle, +} from "node:fs/promises"; +import { join } from "node:path"; + +import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js"; +import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { DECOMPILE_DIRECTORY_JOURNAL_NAME } from "../../contracts/decompileDirectory.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { OwnedStagedOutput } from "../../filesystem/OwnedStagedOutput.js"; +import { + hashStableFile, + stableFileHashCause, +} from "../../filesystem/StableFileHash.js"; +import type { OwnedLease } from "../../process/OwnedLease.js"; +import type { RetainedResourceScope } from "../AnalysisProvider.js"; + +/** Own every descriptor and staging area until cleanup, then release the directory lease. */ +export class ExportResources { + readonly #files = new ArtifactResourceScope(); + readonly #cleanup: AnalysisCleanupLease; + #removeRoot = false; + #journalIdentity: FileIdentity | undefined; + #rootRemoved = false; + #retained = false; + #functionsRemoved = false; + #journalRemoved = false; + #leaseReleased = false; + #functionsIdentity: FileIdentity | undefined; + + constructor( + readonly root: string, + private readonly identity: FileIdentity, + private readonly lease: OwnedLease, + private readonly scope: RetainedResourceScope | undefined, + ) { + const resources = this; + this.#cleanup = AnalysisCleanupLease.forCloseOwner( + { + close: async () => { + await this.#files.retryCleanup(); + if (this.#removeRoot && !this.#rootRemoved) + await this.#removeCreatedRoot(); + await this.lease.release(); + this.#leaseReleased = true; + }, + get cleanupResources() { + return resources.cleanupResources; + }, + }, + root, + ); + } + + get cleanupResources(): readonly string[] { + return [ + ...this.#files.cleanupResources, + ...(this.#removeRoot && !this.#rootRemoved ? [this.root] : []), + ...(this.#leaseReleased ? [] : [this.lease.socketPath ?? this.root]), + ]; + } + + ownFunctionsDirectory(identity: FileIdentity): void { + this.#functionsIdentity = identity; + } + + ownFile(handle: FileHandle, path: string, journal = false) { + const file = new OwnedFileHandle(handle); + const owner = { + kind: "lease" as const, + resource: path, + lease: { + release: async () => { + if (journal && handle.fd >= 0) + this.#journalIdentity = await handle + .stat({ bigint: true }) + .catch(() => undefined); + await file.close(); + }, + }, + }; + this.#files.retain(owner); + return async (): Promise => + (await this.#files.release(owner)).kind === "released"; + } + + /** Check existing output with bounded memory and the shared file identity owner. */ + async hashFile(path: string, signal?: AbortSignal): Promise { + try { + const hash = await hashStableFile(path, { + ...(signal === undefined ? {} : { signal }), + retainFailedClose: (handle) => + this.#files.retain({ kind: "file-handle", resource: path, handle }), + }); + return hash.sha256; + } catch (cause: unknown) { + throw stableFileHashCause(cause); + } + } + + /** Publish only a verified staged file; failed close or removal stays owned. */ + async publish( + bytes: string | Buffer, + destination: string, + mode: "link" | "rename", + ): Promise { + const directory = await mkdtemp(join(this.root, ".decompile-stage-")); + const staging = await OwnedStagedOutput.create(directory); + const owner = { + kind: "staged-output" as const, + resource: directory, + output: staging, + }; + this.#files.retain(owner); + try { + const path = join(directory, "content"); + const handle = await open(path, "wx", 0o600); + staging.setFile(new OwnedFileHandle(handle)); + await handle.writeFile(bytes); + await handle.sync(); + await staging.closeFile(); + await staging.assertReadyForPublication(path); + if (mode === "link") await link(path, destination); + else { + await rename(path, destination); + staging.recordPublishedRename(); + } + } finally { + const released = await this.#files.release(owner); + if (released.kind === "failed") throw released.cause; + } + } + + /** Keep the lease through all initialization cleanup, including any retry. */ + async failedInitialization( + error: AnalysisError, + ): Promise> { + this.#removeRoot = true; + const closed = await this.close(); + return closed.ok + ? err(error) + : err( + analysisErrorWithCleanupFailure( + error, + closed.error, + "decompile_all", + ).bindCleanupDependencies([closed.error]), + ); + } + + async close(): Promise> { + try { + await this.#cleanup.release(); + return ok(undefined); + } catch (cause: unknown) { + const error = new AnalysisOutputError( + "decompile_all", + "decompile export cleanup failed", + { + cause, + cleanup: { + reason: "Owned export resources could not be released.", + resources: [this.root], + }, + }, + ); + error.bindCleanupResourceOwner(this.#cleanup); + this.#cleanup.bindCleanupError(error); + if (!this.#retained && this.scope !== undefined) { + this.#retained = true; + const cleanup = this.#cleanup; + this.scope.retain( + { + close: () => cleanup.release(), + get cleanupResources() { + return cleanup.cleanupResources; + }, + }, + this.root, + ); + } + return err(error); + } + } + + async #removeCreatedRoot(): Promise { + const root = await lstat(this.root, { bigint: true }); + if ( + !sameIdentity(root, this.identity) || + !root.isDirectory() || + root.isSymbolicLink() + ) + throw new Error("Export directory identity changed during cleanup"); + // Initialization has not exported functions. Refuse unexpected entries rather + // than recursively deleting work introduced by another owner. + if (!this.#journalRemoved) { + const journal = join(this.root, DECOMPILE_DIRECTORY_JOURNAL_NAME); + const stat = await lstat(journal, { bigint: true }).catch( + (cause: unknown) => { + if (isMissing(cause)) return undefined; + throw cause; + }, + ); + if (stat !== undefined) { + if ( + this.#journalIdentity === undefined || + !sameIdentity(stat, this.#journalIdentity) + ) + throw new Error( + "Cannot establish ownership of the journal during cleanup", + ); + await unlink(journal); + } + this.#journalRemoved = true; + } + if (!this.#functionsRemoved) { + const path = join(this.root, "functions"); + const stat = await lstat(path, { bigint: true }).catch( + (cause: unknown) => { + if (isMissing(cause) && this.#functionsIdentity === undefined) + return undefined; + throw cause; + }, + ); + if (stat !== undefined) { + if ( + this.#functionsIdentity === undefined || + !sameIdentity(stat, this.#functionsIdentity) + ) + throw new Error( + "Functions directory identity changed during cleanup", + ); + await rmdir(path); + } + this.#functionsRemoved = true; + } + await rmdir(this.root); + this.#rootRemoved = true; + } +} + +interface FileIdentity { + readonly dev: bigint; + readonly ino: bigint; +} + +const sameIdentity = (left: FileIdentity, right: FileIdentity): boolean => + left.dev === right.dev && left.ino === right.ino; + +const isMissing = (cause: unknown): boolean => + cause instanceof Error && "code" in cause && cause.code === "ENOENT"; diff --git a/src/application/decompileDirectory/exportTree.ts b/src/application/decompileDirectory/exportTree.ts new file mode 100644 index 0000000000..d6e2b6a259 --- /dev/null +++ b/src/application/decompileDirectory/exportTree.ts @@ -0,0 +1,970 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { + chmod, + lstat, + mkdir, + open, + realpath, + rmdir, + type FileHandle, +} from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; + +import { + DECOMPILE_DIRECTORY_FUNCTIONS_DIRECTORY, + DECOMPILE_DIRECTORY_INDEX_NAME, + DECOMPILE_DIRECTORY_JOURNAL_NAME, + decompileDirectoryIndexSchema, + decompileJournalHeaderSchema, + decompileJournalLineSchema, + isDecompileFunctionRelativePath, + type DecompileDirectoryFunctionRecord, + type DecompileJournalHeader, +} from "../../contracts/decompileDirectory.js"; +import { + AnalysisOutputError, + AnalysisResourceConstraintError, + type AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { isPathWithinRoot } from "../../domain/localPath.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + acquireOwnedLease, + type LeaseFailure, + type OwnedLease, +} from "../../process/OwnedLease.js"; +import type { RetainedResourceScope } from "../AnalysisProvider.js"; +import { ExportResources } from "./exportResources.js"; +import { decompileInputError } from "./inputError.js"; +import { compareProcedureAddresses } from "./selection.js"; + +export interface ExportIdentity { + readonly targetSha256: string | null; + readonly provider: DecompileJournalHeader["provider"]; + readonly analysisProfileDigest: string | null; + readonly layout: DecompileJournalHeader["layout"]; +} + +export type OutputPlacement = + | { readonly mode: "create"; readonly root: string } + | { readonly mode: "resume"; readonly root: string }; + +/** Refuse a bad path before provider work. This does not create a directory. */ +export const classifyOutputPath = async ( + requested: string, + resume: boolean, +): Promise> => { + const stripped = requested.replace(/\/+$/u, ""); + const name = basename(stripped); + if (stripped.length === 0 || name === "" || name === "." || name === "..") + return err( + decompileInputError( + "output_directory", + "The output directory name is invalid.", + ), + ); + const listed = await lstat(stripped).then( + (stat) => stat, + (cause: unknown) => cause, + ); + if (!isDirectoryStat(listed)) { + if (resume && nodeCode(listed) === "ENOENT") + return err( + decompileInputError( + "output_directory", + "Resume requires an existing export directory. Omit resume to start a new export.", + ), + ); + return missingOrRejected(stripped, listed); + } + if (!resume) + return err( + decompileInputError( + "output_directory", + "Output directory already exists. Pass resume to continue an export that matches this target and analysis profile.", + ), + ); + const root = await realpath(stripped).catch((cause: unknown) => cause); + return typeof root === "string" + ? ok({ mode: "resume", root }) + : err( + decompileInputError( + "output_directory", + "The output directory could not be resolved.", + ), + ); +}; + +/** Owns one export directory and its append-only journal. */ +export class DecompileExport { + private constructor( + readonly root: string, + private readonly journal: FileHandle, + private readonly records: Map, + private readonly written: Map, + private readonly paths: PathReservations, + private readonly identity: ExportIdentity, + private readonly resources: ExportResources, + ) {} + + static open(input: { + readonly placement: OutputPlacement; + readonly identity: ExportIdentity; + readonly resourceScope?: RetainedResourceScope | undefined; + }): Promise> { + return input.placement.mode === "create" + ? DecompileExport.#create( + input.placement.root, + input.identity, + input.resourceScope, + ) + : DecompileExport.#resume( + input.placement.root, + input.identity, + input.resourceScope, + ); + } + + /** The latest journal outcome for an address. */ + record(address: string): DecompileDirectoryFunctionRecord | undefined { + return this.records.get(address); + } + + /** + * The last successful write for an address. A later failure, such as a + * transient read error, does not erase the checksum that verifies the file. + */ + lastWritten(address: string): DecompileDirectoryFunctionRecord | undefined { + return this.written.get(address); + } + + /** + * Stable relative path for an address. The first path the journal gave this + * address wins, even when a later failure record names no path. + */ + pathFor(address: string): string { + const owned = this.paths.byAddress.get(address); + if (owned !== undefined) return owned; + const relativePath = allocateFunctionPath( + address, + this.paths.byPortablePath, + ); + reservePath(this.paths, relativePath, address); + return relativePath; + } + + async append( + record: DecompileDirectoryFunctionRecord, + ): Promise> { + const line = decompileJournalLineSchema.safeParse({ + kind: "function", + ...record, + }); + if (!line.success || line.data.kind !== "function") + return err( + new AnalysisOutputError( + "decompile_all", + "decompile journal record is invalid", + ), + ); + const written = await this.journal + .writeFile(`${JSON.stringify(line.data)}\n`) + .then(() => this.journal.sync()) + .then( + () => true, + () => false, + ); + if (!written) + return err( + new AnalysisOutputError( + "decompile_all", + "decompile journal could not be updated", + ), + ); + // The journal schema owns the parsed snapshot, including nested fields; + // the caller may have changed its input while the append was pending. + const { kind: _kind, ...committed } = line.data; + this.records.set(committed.address, committed); + if (committed.status === "written") + this.written.set(committed.address, committed); + if (committed.relative_path !== undefined) + reservePath(this.paths, committed.relative_path, committed.address); + return ok(undefined); + } + + async writeIndex(input: { + readonly layoutStatus: "address" | "debug_layout_unavailable"; + readonly limitations: readonly string[]; + }): Promise> { + // Appends update this owner only after journal sync; resume parses once. + // Reuse those committed facts instead of re-reading and reinterpreting them. + const index = decompileDirectoryIndexSchema.safeParse({ + version: 1, + target_sha256: this.identity.targetSha256, + provider: this.identity.provider, + analysis_profile_digest: this.identity.analysisProfileDigest, + layout: this.identity.layout, + layout_status: input.layoutStatus, + limitations: [...input.limitations], + functions: [...this.records.values()].sort((left, right) => + compareProcedureAddresses(left.address, right.address), + ), + }); + if (!index.success) + return err( + new AnalysisOutputError( + "decompile_all", + "decompile index could not be built", + ), + ); + return writeOwnedIndex( + this.root, + `${JSON.stringify(index.data, null, 2)}\n`, + this.resources, + ); + } + + commitFunctionFile(relativePath: string, bytes: Buffer): Promise { + return commitFunctionFile(this.root, relativePath, bytes, this.resources); + } + + hashFile(relativePath: string, signal?: AbortSignal): Promise { + return this.resources.hashFile(join(this.root, relativePath), signal); + } + + /** Failed cleanup retains the same files and lease for a later session retry. */ + close(): Promise> { + return this.resources.close(); + } + + static async #create( + root: string, + identity: ExportIdentity, + scope: RetainedResourceScope | undefined, + ): Promise> { + const created = await createPrivateDirectory(root); + if (!created.ok) return created; + // The directory exists before its journal, so a resume that starts in + // between takes the same lease and cannot write alongside this export. + const lease = await acquireExportLease(root); + // Even an empty directory may now have another lease holder preparing its + // journal. Without the lease, preserve it rather than disrupting that run. + if (!lease.ok) { + lease.error.retainDiagnostics({ created_output_directory: root }); + return lease; + } + const resources = new ExportResources( + root, + lease.value.identity, + lease.value.lease, + scope, + ); + const functions = await prepareFunctionsDirectory(root, resources); + if (!functions.ok) return resources.failedInitialization(functions.error); + const opened = await seedJournal(root, identity, resources); + if (!opened.ok) return resources.failedInitialization(opened.error); + return ok( + new DecompileExport( + root, + opened.value, + new Map(), + new Map(), + { byPortablePath: new Map(), byAddress: new Map() }, + identity, + resources, + ), + ); + } + + static async #resume( + root: string, + identity: ExportIdentity, + scope: RetainedResourceScope | undefined, + ): Promise> { + // The lease is held before the journal is read, so no other run appends + // between this read and this run's own appends, repair or index. + const lease = await acquireExportLease(root); + if (!lease.ok) return lease; + const resources = new ExportResources( + root, + lease.value.identity, + lease.value.lease, + scope, + ); + const opened = await openResumedJournal(root, identity, resources); + if (!opened.ok) { + const closed = await resources.close(); + return closed.ok + ? opened + : err( + analysisErrorWithCleanupFailure( + opened.error, + closed.error, + "decompile_all", + ).bindCleanupDependencies([closed.error]), + ); + } + return ok( + new DecompileExport( + root, + opened.value.handle, + opened.value.journal.records, + opened.value.journal.written, + opened.value.journal.paths, + identity, + resources, + ), + ); + } +} + +const openResumedJournal = async ( + root: string, + identity: ExportIdentity, + resources: ExportResources, +): Promise< + Result< + { readonly handle: FileHandle; readonly journal: OpenedJournal }, + AnalysisError + > +> => { + const opened = await readJournal(root, identity, resources); + if (!opened.ok) return opened; + const functions = await prepareFunctionsDirectory(root, resources); + if (!functions.ok) return functions; + const handle = await openJournal(root, "a"); + if (!handle.ok) return handle; + resources.ownFile( + handle.value, + join(root, DECOMPILE_DIRECTORY_JOURNAL_NAME), + true, + ); + // Drop a torn final line left by a crash so the next append starts a line. + const repaired = + opened.value.committedBytes === opened.value.journalBytes || + (await handle.value.truncate(opened.value.committedBytes).then( + () => true, + () => false, + )); + if (!repaired) { + return err( + decompileInputError( + "output_directory", + "The decompile journal could not be repaired after an interrupted write.", + ), + ); + } + return ok({ handle: handle.value, journal: opened.value }); +}; + +export type FileCommit = + | { readonly status: "written"; readonly sha256: string } + | { readonly status: "matched"; readonly sha256: string } + | { readonly status: "conflict" } + | { readonly status: "io" }; + +/** + * Publish bytes with `link`, which fails when the final name exists. + * A matching existing file is adopted. A different file is left unchanged. + */ +const commitFunctionFile = async ( + root: string, + relativePath: string, + bytes: Buffer, + resources: ExportResources, +): Promise => { + if (!isDecompileFunctionRelativePath(relativePath)) return { status: "io" }; + const finalPath = join(root, relativePath); + if (!isPathWithinRoot(root, finalPath)) return { status: "io" }; + const digest = sha256(bytes); + const linked = await resources.publish(bytes, finalPath, "link").then( + () => true, + (cause: unknown) => cause, + ); + if (linked === true) return { status: "written", sha256: digest }; + if (nodeCode(linked) !== "EEXIST") return { status: "io" }; + const existing = await resources + .hashFile(finalPath) + .catch((cause: unknown) => cause); + if (typeof existing !== "string") return { status: "io" }; + return existing === digest + ? { status: "matched", sha256: digest } + : { status: "conflict" }; +}; + +const missingOrRejected = async ( + stripped: string, + listed: unknown, +): Promise> => { + if (isSymlinkStat(listed)) + return err( + decompileInputError( + "output_directory", + "The output directory must be a real directory, not a symlink.", + ), + ); + if (isStat(listed)) + return err( + decompileInputError( + "output_directory", + "The output path exists and is not a directory.", + ), + ); + if (nodeCode(listed) !== "ENOENT") + return err( + decompileInputError( + "output_directory", + "The output directory could not be inspected.", + ), + ); + const parent = await realpath(dirname(stripped)).catch( + (cause: unknown) => cause, + ); + if (typeof parent !== "string") + return err( + decompileInputError( + "output_directory", + "The output directory's parent does not exist.", + ), + ); + return ok({ mode: "create", root: join(parent, basename(stripped)) }); +}; + +const createPrivateDirectory = async ( + root: string, +): Promise> => { + const created = await mkdir(root, { mode: 0o700 }).catch( + (cause: unknown) => cause, + ); + if (created !== undefined) + return err( + nodeCode(created) === "EEXIST" + ? decompileInputError( + "output_directory", + "Output directory already exists. Pass resume to continue an export that matches this target and analysis profile.", + ) + : decompileInputError( + "output_directory", + "The output directory could not be created.", + ), + ); + // This call created the directory, so it owns removing it when the + // directory cannot be made private. Leaving it would block a fresh export + // and a resume alike, because no journal exists yet. + const secured = await chmod(root, 0o700).catch((cause: unknown) => cause); + if (secured === undefined) return ok(undefined); + // The directory is still empty, so a non-recursive removal fails closed on + // a path that another process filled meanwhile or replaced with a file or + // symlink. + const removed = await rmdir(root).then( + () => true, + () => false, + ); + return err( + decompileInputError( + "output_directory", + removed + ? "The output directory could not be secured." + : "The output directory could not be secured, and the directory this call created could not be removed. Remove it before retrying.", + ), + ); +}; + +const prepareFunctionsDirectory = async ( + root: string, + resources: ExportResources, +): Promise> => { + const directory = join(root, DECOMPILE_DIRECTORY_FUNCTIONS_DIRECTORY); + const created = await mkdir(directory, { mode: 0o700 }).catch( + (cause: unknown) => cause, + ); + if (created !== undefined && nodeCode(created) !== "EEXIST") + return err( + decompileInputError( + "output_directory", + "The functions directory could not be created.", + ), + ); + const stat = await lstat(directory, { bigint: true }).catch( + (cause: unknown) => cause, + ); + if (!isDirectoryStat(stat) || !hasFileIdentity(stat)) + return err( + decompileInputError( + "output_directory", + "The functions directory is not a real directory.", + ), + ); + resources.ownFunctionsDirectory(stat); + const secured = await chmod(directory, 0o700).catch( + (cause: unknown) => cause, + ); + return secured === undefined + ? ok(undefined) + : err( + decompileInputError( + "output_directory", + "The functions directory could not be secured.", + ), + ); +}; + +const seedJournal = async ( + root: string, + identity: ExportIdentity, + resources: ExportResources, +): Promise> => { + const header = decompileJournalHeaderSchema.safeParse({ + kind: "header", + version: 1, + target_sha256: identity.targetSha256, + provider: identity.provider, + analysis_profile_digest: identity.analysisProfileDigest, + layout: identity.layout, + }); + if (!header.success) + return err( + new AnalysisOutputError( + "decompile_all", + "decompile journal header is invalid", + ), + ); + const opened = await openJournal(root, "wx"); + if (!opened.ok) return opened; + resources.ownFile( + opened.value, + join(root, DECOMPILE_DIRECTORY_JOURNAL_NAME), + true, + ); + const seeded = await opened.value + .writeFile(`${JSON.stringify(header.data)}\n`) + .then(() => opened.value.sync()) + .then(() => chmod(join(root, DECOMPILE_DIRECTORY_JOURNAL_NAME), 0o600)) + .then( + () => true, + () => false, + ); + if (seeded) return opened; + return err( + decompileInputError( + "output_directory", + "The decompile journal could not be created.", + ), + ); +}; + +/** Absent on Windows, where the journal's lstat check is the only guard. */ +const NO_FOLLOW = constants.O_NOFOLLOW ?? 0; + +const JOURNAL_FLAGS = { + wx: constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | NO_FOLLOW, + a: constants.O_WRONLY | constants.O_APPEND | NO_FOLLOW, +} as const; + +/** The journal is never opened through a symlink: repair truncates it. */ +const openJournal = async ( + root: string, + flag: "wx" | "a", +): Promise> => { + const handle = await open( + join(root, DECOMPILE_DIRECTORY_JOURNAL_NAME), + JOURNAL_FLAGS[flag], + 0o600, + ).catch((cause: unknown) => cause); + return isFileHandle(handle) + ? ok(handle) + : err( + decompileInputError( + "output_directory", + "The decompile journal could not be opened.", + ), + ); +}; + +interface OpenedJournal { + readonly records: Map; + readonly written: Map; + readonly paths: PathReservations; + /** Byte length of the newline-terminated lines. */ + readonly committedBytes: number; + /** Byte length of the journal file as read. */ + readonly journalBytes: number; +} + +/** Read the journal only when it is a regular file, not a symlink. */ +const readJournalBytes = async ( + root: string, + resources: ExportResources, +): Promise => { + const path = join(root, DECOMPILE_DIRECTORY_JOURNAL_NAME); + const stat = await lstat(path).catch((cause: unknown) => cause); + if (!isStat(stat)) return stat; + if (!stat.isFile()) return new Error("journal is not a regular file"); + const handle = await open(path, constants.O_RDONLY | NO_FOLLOW).catch( + (cause: unknown) => cause, + ); + if (!isFileHandle(handle)) return handle; + const close = resources.ownFile(handle, path); + const bytes = await handle.readFile().catch((cause: unknown) => cause); + return (await close()) + ? bytes + : new Error("decompile journal read handle could not be closed"); +}; + +const readJournal = async ( + root: string, + identity: ExportIdentity, + resources: ExportResources, +): Promise> => { + const bytes = await readJournalBytes(root, resources); + if (!(bytes instanceof Buffer)) + return err( + nodeCode(bytes) === "ENOENT" + ? decompileInputError( + "output_directory", + "Resume requires decompile-journal.jsonl in the output directory. An export stopped before its journal was written has no functions to keep; choose a new output directory, or remove this one if it is yours.", + ) + : decompileInputError( + "output_directory", + "The decompile journal could not be read.", + ), + ); + const parsed = parseJournal(bytes); + if (!parsed.ok) return parsed; + const unknown = unknownIdentity(parsed.value.header, identity); + if (unknown !== undefined) return err(unknown); + return sameIdentity(parsed.value.header, identity) + ? ok({ + records: parsed.value.records, + written: parsed.value.written, + paths: parsed.value.paths, + committedBytes: parsed.value.committedBytes, + journalBytes: parsed.value.journalBytes, + }) + : err( + decompileInputError( + "output_directory", + "The decompile journal does not match this target, provider, analysis profile, or layout.", + ), + ); +}; + +const parseJournal = ( + bytes: Buffer, +): Result< + OpenedJournal & { readonly header: DecompileJournalHeader }, + AnalysisError +> => { + // Every committed line ends with a newline. Bytes after the last newline are + // an interrupted append and are not part of the journal. Offsets are bytes, + // so damaged UTF-8 cannot move the truncation point. + const committedBytes = bytes.lastIndexOf(0x0a) + 1; + const lines = bytes.subarray(0, committedBytes).toString("utf8").split("\n"); + lines.pop(); + const header = decompileJournalHeaderSchema.safeParse(parseJson(lines[0])); + if (!header.success) return err(incompatibleJournal()); + const records = new Map(); + const written = new Map(); + // A path stays reserved by the first address that used it, even when that + // address's latest record no longer names it. + const paths: PathReservations = { + byPortablePath: new Map(), + byAddress: new Map(), + }; + for (const line of lines.slice(1)) { + const parsed = decompileJournalLineSchema.safeParse(parseJson(line)); + if (!parsed.success || parsed.data.kind !== "function") + return err(incompatibleJournal()); + const { kind: _kind, ...record } = parsed.data; + const path = record.relative_path; + if (path !== undefined) { + const owner = paths.byPortablePath.get(portablePathKey(path)); + if (owner !== undefined && owner !== record.address) + return err(incompatibleJournal()); + reservePath(paths, path, record.address); + } + records.set(record.address, record); + if (record.status === "written") written.set(record.address, record); + } + return ok({ + header: header.data, + records, + written, + paths, + committedBytes, + journalBytes: bytes.length, + }); +}; + +/** Resume needs both identities known. Two unknown digests are not a match. */ +const unknownIdentity = ( + header: DecompileJournalHeader, + identity: ExportIdentity, +): AnalysisInputError | undefined => { + if (header.target_sha256 === null || identity.targetSha256 === null) + return decompileInputError( + "output_directory", + "Resume requires an observed target SHA-256 for both the journal and this target.", + ); + if ( + header.analysis_profile_digest === null || + identity.analysisProfileDigest === null + ) + return decompileInputError( + "output_directory", + "Resume requires an analysis profile digest for both the journal and this provider.", + ); + return undefined; +}; + +const sameIdentity = ( + header: DecompileJournalHeader, + identity: ExportIdentity, +): boolean => + header.target_sha256 === identity.targetSha256 && + header.analysis_profile_digest === identity.analysisProfileDigest && + header.layout === identity.layout && + header.provider.id === identity.provider.id && + header.provider.name === identity.provider.name && + header.provider.version === identity.provider.version; + +const incompatibleJournal = (): AnalysisInputError => + decompileInputError( + "output_directory", + "The decompile journal is not a compatible export.", + ); + +const parseJson = (line: string | undefined): unknown => { + if (line === undefined) return undefined; + try { + return JSON.parse(line) as unknown; + } catch { + return undefined; + } +}; + +/** Function file paths: each path has one owning address, each address keeps its first path. */ +interface PathReservations { + readonly byPortablePath: Map; + readonly byAddress: Map; +} + +/** Generated and admitted function paths are ASCII; fold only their lookup key. */ +const portablePathKey = (path: string): string => path.toLowerCase(); + +const reservePath = ( + paths: PathReservations, + path: string, + address: string, +): void => { + const key = portablePathKey(path); + if (!paths.byPortablePath.has(key)) paths.byPortablePath.set(key, address); + if (!paths.byAddress.has(address)) paths.byAddress.set(address, path); +}; + +const allocateFunctionPath = ( + address: string, + used: ReadonlyMap, +): string => { + const base = sanitizeAddress(address); + const directory = DECOMPILE_DIRECTORY_FUNCTIONS_DIRECTORY; + const plain = `${directory}/${base}.c`; + if (!used.has(portablePathKey(plain))) return plain; + const suffix = createHash("sha256").update(address).digest("hex").slice(0, 8); + let candidate = `${directory}/${base}-${suffix}.c`; + let discriminator = 0; + // Terminates: `used` is finite. + while (used.has(portablePathKey(candidate))) { + discriminator += 1; + candidate = `${directory}/${base}-${suffix}-${discriminator.toString(16)}.c`; + } + return candidate; +}; + +const sanitizeAddress = (address: string): string => { + const cleaned = address + .replace(/[^A-Za-z0-9._-]+/gu, "_") + .replace(/^\.+/u, "") + .slice(0, 120); + return cleaned.length === 0 ? "function" : cleaned; +}; + +/** Private per-user parent of export lease sockets; it outlives individual runs. */ +const DECOMPILE_EXPORT_LEASE_DIRECTORY = join( + "/tmp", + `rea-decompile-${(process.getuid?.() ?? 0).toString()}`, +); + +/** + * Hold the output directory for this run. The lease names the directory by + * device and inode, so two spellings of one directory share it, and it ends + * when this process exits, so a crashed run leaves no lock behind. + */ +const acquireExportLease = async ( + root: string, +): Promise< + Result< + { + readonly lease: OwnedLease; + readonly identity: { readonly dev: bigint; readonly ino: bigint }; + }, + AnalysisError + > +> => { + const stat = await lstat(root, { bigint: true }).catch( + (cause: unknown) => cause, + ); + if (!isDirectoryStat(stat) || !hasFileIdentity(stat)) + return err( + decompileInputError( + "output_directory", + "The output directory could not be inspected.", + ), + ); + try { + const acquisition = await acquireOwnedLease({ + identity: { + operation: "decompile_all", + device: stat.dev.toString(), + inode: stat.ino.toString(), + }, + runId: randomUUID(), + directory: DECOMPILE_EXPORT_LEASE_DIRECTORY, + failure: exportLeaseFailure, + }); + return acquisition.acquired + ? ok({ lease: acquisition.lease, identity: stat }) + : err( + new AnalysisResourceConstraintError( + "decompile_all", + "capacity", + `another decompile_all run (process ${acquisition.owner.processId.toString()}) is using this output directory`, + { holder_process_id: acquisition.owner.processId }, + { + retryable: true, + remediationAction: + "Wait for the other decompile_all run on this output directory to finish, then retry with resume.", + }, + ), + ); + } catch (cause: unknown) { + return err( + cause instanceof AnalysisError + ? cause + : new AnalysisOutputError( + "decompile_all", + "output directory lease could not be acquired", + { cause }, + ), + ); + } +}; + +const exportLeaseFailure = ( + failure: LeaseFailure, + cause?: unknown, +): AnalysisOutputError => + new AnalysisOutputError( + "decompile_all", + "output directory lease could not be acquired", + { + ...(cause === undefined ? {} : { cause }), + userMessage: exportLeaseMessage(failure), + }, + ); + +const exportLeaseMessage = (failure: LeaseFailure): string => { + switch (failure.kind) { + case "owner_unverified": + return `Another process holds this output directory's lease but did not identify itself (${failure.reason}). Retry after it finishes.`; + case "recovery_busy": + return `Another run is recovering this output directory's lease at ${JSON.stringify(failure.path)}. Retry shortly. If no REA process is running, remove that directory.`; + case "recovery_remove": + return `REA could not remove its lease recovery directory at ${JSON.stringify(failure.path)}. Check its write access before retrying.`; + case "unrecoverable": + return "REA could not take this output directory's lease. Retry after other decompile_all runs finish."; + case "directory_access_denied": + return `REA could not access its lease directory at ${JSON.stringify(failure.directory)}. Restore owner-only permissions (0700) on a directory you own, then retry.`; + case "directory_create": + case "directory_not_private": + case "directory_permissions": + case "directory_inspect": + case "recovery_create": + return `REA could not use its private lease directory ${JSON.stringify(DECOMPILE_EXPORT_LEASE_DIRECTORY)}. Check that it is a directory owned by this user with permissions 0700.`; + } +}; + +const hasFileIdentity = ( + value: unknown, +): value is { readonly dev: bigint; readonly ino: bigint } => + typeof value === "object" && + value !== null && + "dev" in value && + "ino" in value && + typeof value.dev === "bigint" && + typeof value.ino === "bigint"; + +const writeOwnedIndex = async ( + root: string, + text: string, + resources: ExportResources, +): Promise> => { + try { + await resources.publish( + text, + join(root, DECOMPILE_DIRECTORY_INDEX_NAME), + "rename", + ); + return ok(undefined); + } catch (cause: unknown) { + return err( + new AnalysisOutputError( + "decompile_all", + "decompile index could not be written", + { cause }, + ), + ); + } +}; + +export const sha256 = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +const isFileHandle = (value: unknown): value is FileHandle => + typeof value === "object" && + value !== null && + "write" in value && + "sync" in value && + "close" in value; + +const isDirectoryStat = (value: unknown): boolean => + isStat(value) && value.isDirectory() && !value.isSymbolicLink(); + +const isSymlinkStat = (value: unknown): boolean => + isStat(value) && value.isSymbolicLink(); + +const isStat = ( + value: unknown, +): value is { + isDirectory: () => boolean; + isFile: () => boolean; + isSymbolicLink: () => boolean; +} => + typeof value === "object" && + value !== null && + "isDirectory" in value && + "isFile" in value && + "isSymbolicLink" in value && + typeof value.isDirectory === "function" && + typeof value.isSymbolicLink === "function"; + +const nodeCode = (cause: unknown): string | undefined => + typeof cause === "object" && + cause !== null && + "code" in cause && + typeof cause.code === "string" + ? cause.code + : undefined; diff --git a/src/application/decompileDirectory/functionFile.ts b/src/application/decompileDirectory/functionFile.ts new file mode 100644 index 0000000000..94896f18e5 --- /dev/null +++ b/src/application/decompileDirectory/functionFile.ts @@ -0,0 +1,85 @@ +import type { DecompileDirectoryFunctionRecord } from "../../contracts/decompileDirectory.js"; +import type { ExportIdentity } from "./exportTree.js"; + +export interface FunctionBodyFact { + readonly available: false; + readonly reason: string; +} + +export interface FunctionBodyRanges { + readonly available: true; + readonly ranges: readonly { readonly start: string; readonly end: string }[]; + readonly non_contiguous: boolean; +} + +export type FunctionBody = FunctionBodyFact | FunctionBodyRanges; + +/** Line-comment header plus the provider text. A block comment would close early inside pseudocode. */ +export const functionFileText = (input: { + readonly procedure: { + readonly address: string; + readonly name: string | null; + }; + readonly body: FunctionBody; + readonly identity: ExportIdentity; + readonly limitations: readonly string[]; + readonly pseudocode: string; +}): string => { + const lines = [ + "// rea decompile_all", + `// name: ${headerValue(input.procedure.name ?? "")}`, + `// address: ${headerValue(input.procedure.address)}`, + ...bodyLines(input.body), + `// provider: ${headerValue(providerLine(input.identity))}`, + `// analysis_profile: ${input.identity.analysisProfileDigest ?? "unavailable"}`, + ...limitationLines(input.limitations), + "// pseudocode: provider-specific; not original source", + ]; + return `${lines.join("\n")}\n\n${input.pseudocode}`; +}; + +export const bodyRecord = ( + body: FunctionBody, +): DecompileDirectoryFunctionRecord["body"] => + body.available + ? { + available: true, + ranges: body.ranges.map((range) => ({ + start: range.start, + end: range.end, + })), + non_contiguous: body.non_contiguous, + } + : { available: false, reason: body.reason }; + +const bodyLines = (body: FunctionBody): readonly string[] => + body.available + ? [ + "// body: available", + ...body.ranges.map( + (range) => + `// range: ${headerValue(range.start)}-${headerValue(range.end)} inclusive`, + ), + `// non_contiguous: ${body.non_contiguous ? "true" : "false"}`, + ] + : ["// body: unavailable", `// body_reason: ${headerValue(body.reason)}`]; + +const limitationLines = (limitations: readonly string[]): readonly string[] => { + const joined = limitations + .map((limitation) => headerValue(limitation)) + .filter((limitation) => limitation.length > 0) + .join("; "); + if (joined.length <= 4_000) + return [`// limitations: ${joined.length === 0 ? "none" : joined}`]; + return [ + `// limitations: ${joined.slice(0, 4_000)}`, + "// limitations_truncated: true; decompile-index.json keeps the complete list", + ]; +}; + +const providerLine = (identity: ExportIdentity): string => + `${identity.provider.name} ${identity.provider.version ?? "unknown"} (${identity.provider.id})`; + +/** Keep a provider string on one header line without control characters. */ +const headerValue = (value: string): string => + value.replace(/[\u0000-\u001f\u007f]/gu, " ").slice(0, 2_000); diff --git a/src/application/decompileDirectory/inputError.ts b/src/application/decompileDirectory/inputError.ts new file mode 100644 index 0000000000..88fb1436ef --- /dev/null +++ b/src/application/decompileDirectory/inputError.ts @@ -0,0 +1,10 @@ +import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; + +/** Caller-facing rejection for one decompile-directory argument. */ +export const decompileInputError = ( + path: string, + message: string, +): AnalysisInputError => + new AnalysisInputError("decompile_all", { userMessage: message }, [ + { path: [path], reason: "invalid_value", message }, + ]); diff --git a/src/application/decompileDirectory/selection.ts b/src/application/decompileDirectory/selection.ts new file mode 100644 index 0000000000..7e6aa74ff9 --- /dev/null +++ b/src/application/decompileDirectory/selection.ts @@ -0,0 +1,169 @@ +import type { DecompileDirectoryInput } from "../../contracts/decompileDirectory.js"; +import { addressCoordinate } from "../../domain/native/addressCoordinate.js"; +import { nativeAddressCoordinateInputSchema } from "../../domain/native/nativeAddress.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { decompileInputError } from "./inputError.js"; + +export interface SelectedProcedure { + readonly address: string; + readonly name: string | null; +} + +interface AddressBound { + readonly space: string | null; + readonly offset: bigint; +} + +export interface DecompileBounds { + readonly start: AddressBound | null; + readonly end: AddressBound | null; +} + +/** Reject address bounds that cannot be compared inside one address space. */ +export const validateDecompileBounds = ( + input: DecompileDirectoryInput, +): Result => { + const start = parseBound(input.address_start, "address_start"); + if (!start.ok) return start; + const end = parseBound(input.address_end, "address_end"); + if (!end.ok) return end; + if ( + start.value !== null && + end.value !== null && + start.value.space !== end.value.space + ) + return err( + decompileInputError( + "address_end", + "Address bounds must use the same address space.", + ), + ); + if ( + start.value !== null && + end.value !== null && + end.value.offset < start.value.offset + ) + return err( + decompileInputError( + "address_end", + "Address end is before address start.", + ), + ); + return ok({ start: start.value, end: end.value }); +}; + +export interface ProcedureSelection { + readonly procedures: readonly SelectedProcedure[]; + readonly excluded: number; + readonly collapsedDuplicates: boolean; +} + +/** Filter and stably order unique inventory rows. Duplicate addresses collapse. */ +export const selectProcedures = ( + rows: readonly SelectedProcedure[], + input: DecompileDirectoryInput, + bounds: DecompileBounds, +): ProcedureSelection => { + const seen = new Set(); + const selected = new Map(); + let collapsedDuplicates = false; + for (const row of rows) { + if (seen.has(row.address)) collapsedDuplicates = true; + seen.add(row.address); + // Every observed label participates in caller filtering before one output + // is chosen for the address. An earlier non-match cannot hide an alias. + if (!selected.has(row.address) && matches(row, input, bounds)) + selected.set(row.address, row); + } + const procedures = [...selected.values()].sort(compareProcedures); + return { + procedures, + excluded: seen.size - procedures.length, + collapsedDuplicates, + }; +}; + +const parseBound = ( + value: string | undefined, + path: "address_start" | "address_end", +): Result => { + if (value === undefined) return ok(null); + const coordinate = nativeAddressCoordinateInputSchema.safeParse(value); + return !coordinate.success + ? err( + decompileInputError( + path, + "Address bounds must be 0xoffset or space:0xoffset.", + ), + ) + : ok(coordinate.data); +}; + +const matches = ( + procedure: SelectedProcedure, + input: DecompileDirectoryInput, + bounds: DecompileBounds, +): boolean => + matchesName(procedure.name, input.name) && + matchesNamespace(procedure.name, input.namespace) && + matchesAddress(procedure.address, bounds); + +const matchesName = ( + name: string | null, + filter: string | undefined, +): boolean => filter === undefined || (name !== null && name.includes(filter)); + +const matchesNamespace = ( + name: string | null, + filter: string | undefined, +): boolean => + filter === undefined || (name !== null && name.startsWith(filter)); + +const matchesAddress = (address: string, bounds: DecompileBounds): boolean => { + if (bounds.start === null && bounds.end === null) return true; + const coordinate = addressCoordinate(address); + if (coordinate === null) return false; + if ( + bounds.start !== null && + (coordinate.space !== bounds.start.space || + coordinate.offset < bounds.start.offset) + ) + return false; + return ( + bounds.end === null || + (coordinate.space === bounds.end.space && + coordinate.offset <= bounds.end.offset) + ); +}; + +const compareProcedures = ( + left: SelectedProcedure, + right: SelectedProcedure, +): number => { + const byAddress = compareProcedureAddresses(left.address, right.address); + if (byAddress !== 0) return byAddress; + const byName = compareUnicodeCodePoints(left.name ?? "", right.name ?? ""); + return byName === 0 + ? compareUnicodeCodePoints(left.address, right.address) + : byName; +}; + +/** Numeric order within an address space; unparsed addresses sort last. */ +export const compareProcedureAddresses = ( + left: string, + right: string, +): number => { + const a = addressCoordinate(left); + const b = addressCoordinate(right); + if (a === null || b === null) { + if (a === null && b === null) return compareUnicodeCodePoints(left, right); + return a === null ? 1 : -1; + } + const space = compareUnicodeCodePoints(a.space ?? "", b.space ?? ""); + if (space !== 0) return space; + if (a.offset < b.offset) return -1; + if (a.offset > b.offset) return 1; + return 0; +}; diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.ts index 74245a5729..91e67d7783 100644 --- a/src/cli/coreBinaryCommands.ts +++ b/src/cli/coreBinaryCommands.ts @@ -17,6 +17,7 @@ import { withCommandCancellation } from "./commandCancellation.js"; import { javascriptApplicationOptions } from "../cliObservationOptions.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; +import { registerDecompileDirectoryCommand } from "./decompileDirectoryCommand.js"; /** Register provider-neutral binary overview and procedure CLI commands. */ export const registerCoreBinaryCommands = ( @@ -27,6 +28,7 @@ export const registerCoreBinaryCommands = ( ): void => { registerOverviewCommands(cli, logger, runDirectAnalysis, environment); registerDecompileCommand(cli, logger, runDirectAnalysis); + registerDecompileDirectoryCommand(cli, logger, runDirectAnalysis); registerFunctionCommand(cli, logger, runDirectAnalysis); registerInstructionsCommand(cli, logger, runDirectAnalysis); registerSearchCommand(cli, logger, runDirectAnalysis); diff --git a/src/cli/decompileDirectoryCommand.ts b/src/cli/decompileDirectoryCommand.ts new file mode 100644 index 0000000000..2cf651f1cf --- /dev/null +++ b/src/cli/decompileDirectoryCommand.ts @@ -0,0 +1,119 @@ +import { z } from "incur"; + +import type { DirectAnalysis } from "../composition/directAnalysis.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { logCliCommand } from "../cliLogging.js"; +import type { Logger } from "pino"; +import { + directAnalysisOptionsFromCli, + nativeOpenOptionFields, +} from "./options.js"; +import type { CliInstance } from "./types.js"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; +import type { JsonValue } from "../domain/jsonValue.js"; + +/** Register `rea decompile-all`, the directory export of provider pseudocode. */ +export const registerDecompileDirectoryCommand = ( + cli: CliInstance, + logger: Logger, + runDirectAnalysis: DirectAnalysis["runDirectAnalysis"], +): void => { + cli.command(CLI_COMMANDS.decompileAll, { + description: + "Write every function's pseudocode into a directory and return a manifest", + args: z.object({ + path: z.string().describe("App or program path"), + }), + options: z.object({ + output: z + .string() + .min(1) + .describe( + "Output directory. A relative path is resolved from the working directory. The parent must exist.", + ), + layout: z + .enum(["address", "dwarf"]) + .default("address") + .describe( + "address names files by function address. dwarf reports that compilation-unit paths are unavailable.", + ), + resume: z + .boolean() + .default(false) + .describe( + "Continue a matching export. Never overwrites a function file.", + ), + name: z + .string() + .min(1) + .optional() + .describe("Case-sensitive literal substring of the function name"), + namespace: z + .string() + .min(1) + .optional() + .describe( + "Case-sensitive function-name prefix, including any separator", + ), + addressStart: z + .string() + .min(1) + .optional() + .describe("Inclusive lower bound, 0xoffset or space:0xoffset"), + addressEnd: z + .string() + .min(1) + .optional() + .describe("Inclusive upper bound, 0xoffset or space:0xoffset"), + limit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Maximum functions to attempt after skipping verified files and recorded external functions", + ), + snapshot: z + .string() + .min(1) + .optional() + .describe( + "Load a local analysis snapshot. decompile-all does not update it.", + ), + ...nativeOpenOptionFields, + }), + run: ({ args, options }) => + logCliCommand(logger, "decompile-all", () => + runDirectAnalysis( + args.path, + "decompile_all", + decompileAllArguments(options), + directAnalysisOptionsFromCli(logger, options), + ), + ), + }); +}; + +const decompileAllArguments = (options: { + readonly output: string; + readonly layout: "address" | "dwarf"; + readonly resume: boolean; + readonly name?: string | undefined; + readonly namespace?: string | undefined; + readonly addressStart?: string | undefined; + readonly addressEnd?: string | undefined; + readonly limit?: number | undefined; +}): Readonly> => ({ + output_directory: absoluteCallerPath(options.output), + layout: options.layout, + resume: options.resume, + ...(options.name === undefined ? {} : { name: options.name }), + ...(options.namespace === undefined ? {} : { namespace: options.namespace }), + ...(options.addressStart === undefined + ? {} + : { address_start: options.addressStart }), + ...(options.addressEnd === undefined + ? {} + : { address_end: options.addressEnd }), + ...(options.limit === undefined ? {} : { limit: options.limit }), +}); diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 3a47cfd3de..1a450389c0 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -18,6 +18,7 @@ export const CLI_COMMANDS = Object.freeze({ analyze: "analyze", inspect: "inspect", decompile: "decompile", + decompileAll: "decompile-all", setup: "setup", doctor: "doctor", uninstall: "uninstall", diff --git a/src/contracts/decompileDirectory.ts b/src/contracts/decompileDirectory.ts new file mode 100644 index 0000000000..bb584f8437 --- /dev/null +++ b/src/contracts/decompileDirectory.ts @@ -0,0 +1,329 @@ +import { z } from "zod"; + +import { digestSchema } from "../domain/digests.js"; +import { absoluteLocalPathStringSchema } from "../domain/localPath.js"; + +/** On-disk journal. Append-only so a crash keeps committed function files. */ +export const DECOMPILE_DIRECTORY_JOURNAL_NAME = "decompile-journal.jsonl"; + +/** Derived catalog of the journal. It never contains pseudocode. */ +export const DECOMPILE_DIRECTORY_INDEX_NAME = "decompile-index.json"; + +/** Directory of one `.c` file per function, named from the canonical address. */ +export const DECOMPILE_DIRECTORY_FUNCTIONS_DIRECTORY = "functions"; + +/** First supported journal version. */ +export const DECOMPILE_DIRECTORY_VERSION = 1; + +/** Failure reasons returned inline. The index keeps every reason. */ +export const DECOMPILE_DIRECTORY_INLINE_FAILURE_LIMIT = 32; + +/** JSON-encoded UTF-8 bytes for one inline failure-detail array. */ +export const DECOMPILE_DIRECTORY_INLINE_FAILURE_BYTES = 2_048; + +const layoutSchema = z.enum(["address", "dwarf"]); + +const exportProviderSchema = z.strictObject({ + id: z.string().min(1), + name: z.string().min(1), + version: z.string().min(1).nullable(), +}); + +const bodySchema = z.discriminatedUnion("available", [ + z.strictObject({ + available: z.literal(false), + reason: z.string().min(1), + }), + z.strictObject({ + available: z.literal(true), + ranges: z.array( + z.strictObject({ + start: z.string().min(1), + end: z + .string() + .min(1) + .describe("Inclusive last address of this body range."), + }), + ), + non_contiguous: z.boolean(), + }), +]); + +/** + * written: a file holds the pseudocode. failed: no file, with a reason. + * external: procedure_info classifies the function as external. No file is + * written and no pseudocode is requested; the body fact is kept as observed. + */ +const functionStatusSchema = z.enum(["written", "failed", "external"]); + +const failureFields = { + address: z.string().min(1), + code: z.string().min(1), + message: z.string().min(1), +}; + +const failureSchema = z.union([ + z.strictObject({ + ...failureFields, + name: z + .string() + .nullable() + .describe("Observed name, or null when unknown."), + }), + z + .strictObject(failureFields) + .describe( + "Name omitted to fit the inline text budget; the index retains the full observed name.", + ), +]); + +export type DecompileDirectoryInlineFailure = z.infer; + +const countSchema = (description: string) => + z.number().int().min(0).describe(description); + +/** + * Caller input for writing pseudocode to a directory. + * The workflow parses this once at the CLI and MCP boundaries. + */ +export const decompileDirectoryInputSchema = z.strictObject({ + output_directory: absoluteLocalPathStringSchema.describe( + "Absolute output directory. The parent must exist. The directory itself must be absent unless resume is true. One run at a time may use a directory; a second run is refused while the first holds it.", + ), + layout: layoutSchema + .default("address") + .describe( + "address names files by canonical address. dwarf asks for compilation-unit paths and is reported unavailable when the provider has no such fact.", + ), + resume: z + .boolean() + .default(false) + .describe( + "Continue a journal for the same target SHA-256, provider, analysis profile, and layout. Both digests must be known. Verified files and functions recorded as external are skipped, conflicting files are left unchanged, and functions not yet tried run before retries of recorded failures.", + ), + name: z + .string() + .min(1) + .optional() + .describe( + "Case-sensitive literal substring of the provider function name.", + ), + namespace: z + .string() + .min(1) + .optional() + .describe( + "Case-sensitive prefix of the provider function name, including any separator such as Foo:: or Class.", + ), + address_start: z + .string() + .min(1) + .optional() + .describe( + "Inclusive lower bound as 0xoffset or space:0xoffset. Must use the same address space as address_end.", + ), + address_end: z + .string() + .min(1) + .optional() + .describe( + "Inclusive upper bound as 0xoffset or space:0xoffset. Must use the same address space as address_start.", + ), + limit: z + .number() + .int() + .min(1) + .optional() + .describe( + "Maximum functions to attempt on this call after skipping verified files and functions recorded as external. Further matches remain for a later resume.", + ), +}); + +export type DecompileDirectoryInput = z.output< + typeof decompileDirectoryInputSchema +>; + +/** Small tool result. Pseudocode stays in the output files. */ +export const decompileDirectoryManifestSchema = z + .strictObject({ + output_directory: z + .string() + .min(1) + .describe("Absolute directory containing the function files."), + index_path: z + .string() + .min(1) + .describe( + "Absolute path of the index. The index lists functions and does not contain pseudocode.", + ), + layout: layoutSchema, + layout_status: z + .enum(["address", "debug_layout_unavailable"]) + .describe( + "address when files are named by address on request. debug_layout_unavailable when dwarf was requested and no compilation-unit fact exists.", + ), + counts: z.strictObject({ + selected: countSchema("Functions matching the filter."), + excluded: countSchema("Inventory functions excluded by the filter."), + requested: countSchema("Functions this call attempted."), + written: countSchema("Function files created by this call."), + failed: countSchema("Functions this call could not write."), + external: countSchema( + "Functions this call found procedure_info classifies as external. No file is written and no pseudocode is requested. A function without a matching procedure_info classification is decompiled as usual.", + ), + skipped: countSchema( + "Selected functions already settled: existing files that still hold the same bytes, whether recorded in the journal or found on disk, and functions the journal records as external.", + ), + remaining: countSchema( + "Selected functions not attempted because the limit was reached.", + ), + }), + failures: z + .array(failureSchema) + .max(DECOMPILE_DIRECTORY_INLINE_FAILURE_LIMIT) + .describe( + "At most 32 failure details within 2,048 JSON-encoded UTF-8 bytes. Names may be omitted to preserve addresses and reasons; the index retains complete records.", + ), + failures_omitted: countSchema( + "Failure records omitted from failures because of the inline count or encoded-text budget.", + ), + provider: exportProviderSchema.describe( + "Provider that produced the pseudocode.", + ), + analysis_profile_digest: digestSchema + .nullable() + .describe( + "Provider analysis-profile digest, or null when the provider did not commit one.", + ), + target_sha256: digestSchema + .nullable() + .describe( + "SHA-256 of the analyzed target, or null when the observation has no file subject.", + ), + limitations: z + .array(z.string()) + .describe("Limits of this export. Pseudocode is not included."), + resume: z.boolean().describe("Whether this call continued a journal."), + }) + .superRefine((manifest, context) => { + const { counts } = manifest; + if (counts.written + counts.failed + counts.external !== counts.requested) + context.addIssue({ + code: "custom", + path: ["counts", "requested"], + message: "requested must equal written plus failed plus external", + }); + if ( + counts.requested + counts.skipped + counts.remaining !== + counts.selected + ) + context.addIssue({ + code: "custom", + path: ["counts", "selected"], + message: "selected must equal requested plus skipped plus remaining", + }); + if (manifest.failures.length + manifest.failures_omitted !== counts.failed) + context.addIssue({ + code: "custom", + path: ["failures_omitted"], + message: "failures plus failures_omitted must equal failed", + }); + }); + +/** One latest journal outcome, without pseudocode. */ +export const decompileDirectoryFunctionSchema = z + .strictObject({ + address: z.string().min(1), + name: z.string().nullable(), + status: functionStatusSchema, + relative_path: z + .string() + .min(1) + .refine( + (path) => isDecompileFunctionRelativePath(path), + "Function paths must remain inside functions/.", + ) + .optional(), + sha256: digestSchema.optional(), + body: bodySchema, + failure: z + .strictObject({ + code: z.string().min(1), + message: z.string().min(1), + }) + .optional(), + /** Complete provider caveats of the observations this attempt used. */ + limitations: z.array(z.string().min(1)).min(1).optional(), + }) + .superRefine((record, context) => { + const valid = + record.status === "written" + ? record.relative_path !== undefined && record.sha256 !== undefined + : record.status === "failed" + ? record.failure !== undefined + : record.relative_path === undefined && + record.sha256 === undefined && + record.failure === undefined; + if (!valid) + context.addIssue({ + code: "custom", + path: ["status"], + message: "Function record fields do not match its status.", + }); + }); + +export type DecompileDirectoryFunctionRecord = z.infer< + typeof decompileDirectoryFunctionSchema +>; + +/** Header of a decompile journal. Identity for resume. */ +export const decompileJournalHeaderSchema = z.strictObject({ + kind: z.literal("header"), + version: z.literal(DECOMPILE_DIRECTORY_VERSION), + target_sha256: digestSchema.nullable(), + provider: exportProviderSchema, + analysis_profile_digest: digestSchema.nullable(), + layout: layoutSchema, +}); + +export type DecompileJournalHeader = z.infer< + typeof decompileJournalHeaderSchema +>; + +/** One appended function outcome. */ +export const decompileJournalFunctionSchema = + decompileDirectoryFunctionSchema.safeExtend({ + kind: z.literal("function"), + }); + +export const decompileJournalLineSchema = z.discriminatedUnion("kind", [ + decompileJournalHeaderSchema, + decompileJournalFunctionSchema, +]); + +/** Index rewritten from the journal at the end of a call. */ +export const decompileDirectoryIndexSchema = z.strictObject({ + version: z.literal(DECOMPILE_DIRECTORY_VERSION), + target_sha256: digestSchema.nullable(), + provider: exportProviderSchema, + analysis_profile_digest: digestSchema.nullable(), + layout: layoutSchema, + layout_status: z.enum(["address", "debug_layout_unavailable"]), + limitations: z.array(z.string()), + functions: z.array(decompileDirectoryFunctionSchema), +}); + +/** + * A function file path owned by this export: `functions/.c` with one + * path segment and no traversal. + */ +export const isDecompileFunctionRelativePath = (value: string): boolean => { + if (value.includes("\\") || value.includes("\0")) return false; + const [directory, name, extra] = value.split("/"); + return ( + extra === undefined && + directory === DECOMPILE_DIRECTORY_FUNCTIONS_DIRECTORY && + name !== undefined && + /^[A-Za-z0-9._-]+\.c$/u.test(name) + ); +}; diff --git a/src/contracts/enhancedInputs.ts b/src/contracts/enhancedInputs.ts index 1d715e90ee..4c85e2937e 100644 --- a/src/contracts/enhancedInputs.ts +++ b/src/contracts/enhancedInputs.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { nativeDispatchMetadataInputSchema } from "../domain/native/objcSwiftMetadata.js"; import { nativeProcedureInputSchema } from "../domain/native/nativeProcedureInput.js"; +import { decompileDirectoryInputSchema } from "./decompileDirectory.js"; import { nativeValueTraceInputSchema } from "../domain/native/nativeValueTrace.js"; const traceLiteralInputSchema = z.strictObject({ query: z.string().min(1), @@ -26,6 +27,7 @@ export const enhancedInputSchemas = { .array(z.string().describe("A provider-normalized procedure address")) .default([]), }), + decompile_all: decompileDirectoryInputSchema, get_call_graph: z.strictObject({ address: z .string() diff --git a/src/contracts/enhancedToolContracts.ts b/src/contracts/enhancedToolContracts.ts index 1d1ae2463f..a6a796e614 100644 --- a/src/contracts/enhancedToolContracts.ts +++ b/src/contracts/enhancedToolContracts.ts @@ -45,6 +45,11 @@ export const ENHANCED_TOOL_CONTRACTS = [ "Decompile each explicit procedure symbol or address concurrently. Preserves each input selector in address and reports the observed canonical entry and entry label in procedure; identity or label failures remain explicit without discarding usable pseudocode. Returns ordered per-item success or error results and aggregate counts.", enhancedInputSchemas.batch_decompile, ), + enhanced( + "decompile_all", + "Write provider pseudocode for every analyzed function, or a filtered subset, into a caller-selected directory and return only a manifest. Each file records the provider function name, canonical address, inclusive body ranges when procedure_info supplies them, and that function's procedure_info and pseudocode limitations. Each function's index record keeps its complete limitations. Evidence keeps at most eight first-observed distinct per-function limitations within an encoded-text budget; the omitted count measures observations, not distinct warnings. The manifest reports counts, the output directory, the index path, at most 32 failure details within 2,048 JSON-encoded UTF-8 bytes (names can be omitted; full names remain in the index), provider identity, and the analysis-profile digest. Pseudocode stays in the files. An existing directory is refused unless resume is set and the journal matches the target SHA-256, provider, and analysis profile; verified files are skipped and conflicting files are left unchanged. layout dwarf reports that compilation-unit paths are unavailable and names files by address. A failure of one function does not stop the others. Functions procedure_info classifies as external are counted as external, get no file, and are not decompiled; a resume skips them. Cancellation leaves files already written.", + enhancedInputSchemas.decompile_all, + ), enhanced( "get_call_graph", "Resolve one symbol or address to its canonical procedure entry, then traverse the bound provider's caller or callee relationships until the reachable graph is exhausted. Every node uses a canonical address and has an ok/error status; failures use safe typed projections. Unresolved indirect calls may be missing and results are not a whole-program CFG.", diff --git a/src/contracts/toolContractExamples.ts b/src/contracts/toolContractExamples.ts index 5cb0e3f25f..a50af46de2 100644 --- a/src/contracts/toolContractExamples.ts +++ b/src/contracts/toolContractExamples.ts @@ -53,6 +53,7 @@ export const TOOL_EXAMPLE_OVERRIDES: Readonly< trace_feature: { query: "license" }, trace_call_path: { start: "0x1000", goal: "0x1100" }, trace_native_ui_action: { action: "buildTapped:" }, + decompile_all: { output_directory: "/tmp/decompile-out" }, open_binary: { path: "/tmp/fixture" }, open_bundle_resources: { path: "/tmp/Fixture.app" }, export_evidence_bundle: { path: "/tmp/evidence.json" }, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 46bc1a4a95..4544816cc0 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -563,6 +563,11 @@ export const TOOL_EFFECTS: Readonly> = { get_objc_classes: evidence, get_objc_protocols: evidence, batch_decompile: evidence, + decompile_all: effects({ + mutatesSession: true, + writesFilesystem: true, + idempotent: false, + }), get_call_graph: evidence, analyze_swift_types: evidence, find_xrefs_to_name: evidence, diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 367c9170d7..b05928bedc 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -78,6 +78,7 @@ import { changedBehaviorResultSchema } from "../domain/changedBehavior.js"; import { callPathResultSchema } from "../domain/callPathSchemas.js"; import { staticRuntimeCorrelationResultSchema } from "../domain/staticRuntimeCorrelation.js"; import { reconstructionVerificationResultSchema } from "../domain/reconstructionVerificationSchemas.js"; +import { decompileDirectoryManifestSchema } from "./decompileDirectory.js"; import { analysisErrorProjectionSchema } from "./errorSchemas.js"; import { relationshipCoverageSchema } from "../domain/relationshipCoverage.js"; import { nativeDispatchMetadataResultSchema } from "../domain/native/objcSwiftMetadata.js"; @@ -314,6 +315,7 @@ export const enhancedOutputSchemas: Readonly> = { trace_native_ui_action: evidenceResultOf(nativeInvestigationTraceSchema), get_objc_classes: objcClassDiscoveryOutput, get_objc_protocols: symbolDiscoveryOutput("protocols"), + decompile_all: evidenceResultOf(decompileDirectoryManifestSchema), batch_decompile: evidenceResultOf( z.object({ items: z.array( diff --git a/src/process/HopperLease.ts b/src/process/HopperLease.ts index 6e60974016..dd4bd6741a 100644 --- a/src/process/HopperLease.ts +++ b/src/process/HopperLease.ts @@ -1,30 +1,16 @@ -import { createHash } from "node:crypto"; -import { chmod, mkdir, open, realpath, rm, rmdir } from "node:fs/promises"; -import { constants as fsConstants } from "node:fs"; -import { - createConnection, - createServer, - type Server, - type Socket, -} from "node:net"; +import { realpath } from "node:fs/promises"; import { join } from "node:path"; import { HopperStartError } from "../domain/hopperErrors.js"; -import { safeParseJson } from "../domain/safeJson.js"; +import { + acquireOwnedLease, + type LeaseAcquisition, + type LeaseFailure, + type OwnedLease, +} from "./OwnedLease.js"; -interface HopperTargetLeaseOwner { - readonly runId: string; - readonly processId: number; -} +export type HopperTargetLease = OwnedLease; -export interface HopperTargetLease { - /** Bound socket identity for cleanup diagnostics; omission remains unknown. */ - readonly socketPath?: string; - readonly release: () => Promise; -} - -export type HopperTargetLeaseAcquisition = - | { readonly acquired: true; readonly lease: HopperTargetLease } - | { readonly acquired: false; readonly owner: HopperTargetLeaseOwner }; +export type HopperTargetLeaseAcquisition = LeaseAcquisition; /** Shared private parent of Hopper lease sockets; it outlives individual sessions. */ export const HOPPER_TARGET_LEASE_DIRECTORY = join( @@ -87,258 +73,44 @@ const acquireLease = async (input: { readonly identity: Readonly>; readonly runId: string; readonly directory?: string; -}): Promise => { - const directory = input.directory ?? HOPPER_TARGET_LEASE_DIRECTORY; - await ensureLeaseDirectory(directory); - const key = createHash("sha256") - .update(JSON.stringify(input.identity)) - .digest("hex") - .slice(0, 32); - const socketPath = join(directory, `${key}.sock`); - const owner = { runId: input.runId, processId: process.pid }; - const acquisition = await tryAcquireLease(socketPath, owner); - if (typeof acquisition !== "string") return acquisition; - - // Serialize stale cleanup through the replacement bind: otherwise two - // recoverers can unlink the socket that one of them has just acquired. - const recoveryPath = `${socketPath}.recovery`; - await reserveRecovery(recoveryPath); - let recovered: HopperTargetLeaseAcquisition | "missing" | "stale" | undefined; - try { - recovered = await tryAcquireLease(socketPath, owner); - if (recovered === "stale") await rm(socketPath, { force: true }); - if (typeof recovered === "string") - recovered = await tryAcquireLease(socketPath, owner); - if (typeof recovered !== "string") return recovered; - throw new HopperStartError({ - userMessage: - "REA could not reserve Hopper for this request. Close competing REA sessions and retry.", - }); - } finally { - try { - await rmdir(recoveryPath); - } catch (cause: unknown) { - if (typeof recovered !== "string" && recovered?.acquired) - await recovered.lease.release(); - throw new HopperStartError({ - cause, - userMessage: `REA could not remove its Hopper recovery directory at ${JSON.stringify(recoveryPath)}. Check this directory's write access before retrying.`, - }); - } - } -}; - -const reserveRecovery = async (recoveryPath: string): Promise => { - try { - await mkdir(recoveryPath, { mode: 0o700 }); - } catch (cause: unknown) { - const busy = - typeof cause === "object" && - cause !== null && - Reflect.get(cause, "code") === "EEXIST"; - throw new HopperStartError({ - cause, - userMessage: busy - ? `REA could not reserve Hopper lease recovery at ${JSON.stringify(recoveryPath)}; the existing socket was left unchanged. Retry after concurrent startup finishes. If recovery was interrupted, stop REA sessions and remove this recovery directory before retrying.` - : `REA could not create its Hopper recovery directory at ${JSON.stringify(recoveryPath)}; the existing socket was left unchanged. Check the parent directory's write access before retrying.`, - }); - } -}; - -const tryAcquireLease = async ( - socketPath: string, - owner: HopperTargetLeaseOwner, -): Promise => { - const server = createServer((socket) => sendOwner(socket, owner)); - // The lease lives exactly as long as its owning process. A lease retained - // after unconfirmed cleanup must not keep a completed one-shot command - // alive; process exit leaves a stale socket that later acquirers recover. - server.unref(); - if (!(await bind(server, socketPath))) { - const existingOwner = await readOwner(socketPath); - return typeof existingOwner === "string" - ? existingOwner - : { acquired: false, owner: existingOwner }; - } - try { - await chmod(socketPath, 0o600); - } catch (cause: unknown) { - await closeServer(server); - throw cause; - } - let released = false; - return { - acquired: true, - lease: { - socketPath, - release: async () => { - if (released) return; - // Node removes a bound Unix socket on close. A second unlink here - // could remove a new owner's socket bound after that close. - await closeServer(server); - released = true; - }, - }, - }; -}; - -const ensureLeaseDirectory = async (directory: string): Promise => { - try { - await mkdir(directory, { recursive: true, mode: 0o700 }); - } catch (cause: unknown) { - throw new HopperStartError({ - cause, - userMessage: "REA could not create its private Hopper lease directory.", - }); - } - // Use O_DIRECTORY|O_NOFOLLOW to open the directory without following symlinks, - // then inspect and secure that descriptor. Later socket operations still use - // the path and are not made atomic by this directory validation. - let handle; - try { - handle = await open( - directory, - fsConstants.O_RDONLY | - (fsConstants.O_DIRECTORY ?? 0) | - (fsConstants.O_NOFOLLOW ?? 0), - ); - const directoryStat = await handle.stat(); - if ( - directoryStat.isSymbolicLink() || - directoryStat.isDirectory() === false || - (process.getuid !== undefined && directoryStat.uid !== process.getuid()) - ) - throw new HopperStartError({ - userMessage: - "REA's Hopper lease directory is not a private directory owned by this user.", - }); - // POSIX permission hardening: chmod on the already-opened descriptor avoids - // a second TOCTOU window where the directory could be swapped between check - // and permission fix. Windows DACL-based security is handled separately. - const posixPermissionsApply = process.getuid !== undefined; - if (posixPermissionsApply && (directoryStat.mode & 0o077) !== 0) { - await handle.chmod(0o700); - // Re-verify permissions on the same opened directory. - const afterChmod = await handle.stat(); - if ( - afterChmod.isSymbolicLink() || - afterChmod.isDirectory() === false || - (afterChmod.mode & 0o077) !== 0 - ) - throw new HopperStartError({ - userMessage: - "REA's Hopper lease directory permissions could not be secured.", - }); - } - } catch (cause: unknown) { - if (cause instanceof HopperStartError) throw cause; - const accessDenied = - process.getuid !== undefined && - cause instanceof Error && - "code" in cause && - cause.code === "EACCES"; - throw new HopperStartError({ - cause, - userMessage: accessDenied - ? `REA could not access its Hopper lease directory at ${JSON.stringify(directory)}. Check its ownership and parent-directory access. For a directory you own, restore owner-only read, write, and search permissions (0700), then retry.` - : "REA could not inspect its private Hopper lease directory.", - }); - } finally { - await handle?.close(); - } -}; - -const bind = (server: Server, socketPath: string): Promise => - new Promise((resolveBind, reject) => { - const onError = (error: NodeJS.ErrnoException): void => { - server.removeListener("listening", onListening); - if (error.code === "EADDRINUSE") resolveBind(false); - else reject(error); - }; - const onListening = (): void => { - server.removeListener("error", onError); - resolveBind(true); - }; - server.once("error", onError); - server.once("listening", onListening); - server.listen(socketPath); +}): Promise => + acquireOwnedLease({ + identity: input.identity, + runId: input.runId, + directory: input.directory ?? HOPPER_TARGET_LEASE_DIRECTORY, + failure: hopperLeaseFailure, }); -const sendOwner = (socket: Socket, owner: HopperTargetLeaseOwner): void => { - socket.end(`${JSON.stringify(owner)}\n`); +const hopperLeaseFailure = ( + failure: LeaseFailure, + cause?: unknown, +): HopperStartError => + new HopperStartError({ + ...(cause === undefined ? {} : { cause }), + userMessage: hopperLeaseMessage(failure), + }); + +const hopperLeaseMessage = (failure: LeaseFailure): string => { + switch (failure.kind) { + case "directory_create": + return "REA could not create its private Hopper lease directory."; + case "directory_not_private": + return "REA's Hopper lease directory is not a private directory owned by this user."; + case "directory_permissions": + return "REA's Hopper lease directory permissions could not be secured."; + case "directory_access_denied": + return `REA could not access its Hopper lease directory at ${JSON.stringify(failure.directory)}. Check its ownership and parent-directory access. For a directory you own, restore owner-only read, write, and search permissions (0700), then retry.`; + case "directory_inspect": + return "REA could not inspect its private Hopper lease directory."; + case "recovery_busy": + return `REA could not reserve Hopper lease recovery at ${JSON.stringify(failure.path)}; the existing socket was left unchanged. Retry after concurrent startup finishes. If recovery was interrupted, stop REA sessions and remove this recovery directory before retrying.`; + case "recovery_create": + return `REA could not create its Hopper recovery directory at ${JSON.stringify(failure.path)}; the existing socket was left unchanged. Check the parent directory's write access before retrying.`; + case "recovery_remove": + return `REA could not remove its Hopper recovery directory at ${JSON.stringify(failure.path)}. Check this directory's write access before retrying.`; + case "owner_unverified": + return `REA could not verify the existing Hopper lease (${failure.reason}); it was left unchanged. Retry when the owning REA session is responsive.`; + case "unrecoverable": + return "REA could not reserve Hopper for this request. Close competing REA sessions and retry."; + } }; - -const readOwner = ( - socketPath: string, -): Promise => - new Promise((resolveOwner, rejectOwner) => { - const socket = createConnection(socketPath); - socket.setEncoding("utf8"); - let response = ""; - let settled = false; - const fail = (reason: string, cause?: unknown): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - socket.destroy(); - rejectOwner( - new HopperStartError({ - cause, - userMessage: `REA could not verify the existing Hopper lease (${reason}); it was left unchanged. Retry when the owning REA session is responsive.`, - }), - ); - }; - const timer = setTimeout(() => fail("owner response timed out"), 250); - const finish = ( - owner: HopperTargetLeaseOwner | "missing" | "stale", - ): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - socket.destroy(); - resolveOwner(owner); - }; - socket.on("data", (chunk: string) => { - response += chunk; - const newline = response.indexOf("\n"); - if (newline < 0) return; - const parsed = safeParseJson(response.slice(0, newline)); - if (!parsed.ok) { - fail("malformed owner response"); - return; - } - const value: unknown = parsed.value; - const runId = - typeof value === "object" && value !== null - ? Reflect.get(value, "runId") - : undefined; - const processId = - typeof value === "object" && value !== null - ? Reflect.get(value, "processId") - : undefined; - if ( - typeof runId === "string" && - runId.length > 0 && - typeof processId === "number" && - Number.isSafeInteger(processId) && - processId > 0 - ) - finish({ runId, processId }); - else fail("invalid owner identity"); - }); - socket.on("error", (cause: NodeJS.ErrnoException) => { - if (cause.code === "ENOENT") finish("missing"); - else if (cause.code === "ECONNREFUSED") finish("stale"); - else fail(cause.code ?? "owner connection failed", cause); - }); - socket.on("end", () => fail("owner closed without a complete response")); - }); - -const closeServer = (server: Server): Promise => - new Promise((resolveClose, reject) => { - if (!server.listening) { - resolveClose(); - return; - } - server.close((error) => (error ? reject(error) : resolveClose())); - }); diff --git a/src/process/OwnedLease.ts b/src/process/OwnedLease.ts new file mode 100644 index 0000000000..425f40ace9 --- /dev/null +++ b/src/process/OwnedLease.ts @@ -0,0 +1,320 @@ +import { createHash } from "node:crypto"; +import { chmod, mkdir, open, rm, rmdir } from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; +import { + createConnection, + createServer, + type Server, + type Socket, +} from "node:net"; +import { join } from "node:path"; + +import { safeParseJson } from "../domain/safeJson.js"; + +/** The process that holds a lease, as that process reports itself. */ +export interface LeaseOwner { + readonly runId: string; + readonly processId: number; +} + +export interface OwnedLease { + /** Bound socket identity for cleanup diagnostics; omission remains unknown. */ + readonly socketPath?: string; + readonly release: () => Promise; +} + +export type LeaseAcquisition = + | { readonly acquired: true; readonly lease: OwnedLease } + | { readonly acquired: false; readonly owner: LeaseOwner }; + +/** Why a lease could be neither acquired nor attributed to its owner. */ +export type LeaseFailure = + | { readonly kind: "directory_create" } + | { readonly kind: "directory_not_private" } + | { readonly kind: "directory_permissions" } + | { readonly kind: "directory_access_denied"; readonly directory: string } + | { readonly kind: "directory_inspect" } + | { readonly kind: "recovery_busy"; readonly path: string } + | { readonly kind: "recovery_create"; readonly path: string } + | { readonly kind: "recovery_remove"; readonly path: string } + | { readonly kind: "owner_unverified"; readonly reason: string } + | { readonly kind: "unrecoverable" }; + +export type LeaseFailureFactory = ( + failure: LeaseFailure, + cause?: unknown, +) => Error; + +/** + * Hold one named lease for as long as this process keeps it. + * + * The lease is a listening local socket, so the operating system ends it when + * the owning process exits. A crashed owner leaves a socket that refuses + * connections; recovery replaces only that stale socket. A live owner answers + * with its own identity, which is reported instead of a PID guess. + */ +export const acquireOwnedLease = async (input: { + readonly identity: Readonly>; + readonly runId: string; + /** Private per-user directory for POSIX sockets. Unused on Windows. */ + readonly directory: string; + readonly failure: LeaseFailureFactory; +}): Promise => { + const produced = new WeakSet(); + const fail = (failure: LeaseFailure, cause?: unknown): Error => { + const error = input.failure(failure, cause); + produced.add(error); + return error; + }; + const key = createHash("sha256") + .update(JSON.stringify(input.identity)) + .digest("hex") + .slice(0, 32); + const owner = { runId: input.runId, processId: process.pid }; + if (process.platform === "win32") { + // A named pipe disappears with its last handle, so no stale pipe needs + // recovery. A pipe that vanished between bind and owner lookup is retried. + const pipePath = `\\\\.\\pipe\\rea-lease-${key}`; + for (let attempt = 0; attempt < 2; attempt += 1) { + const acquisition = await tryAcquireLease(pipePath, owner, fail, false); + if (typeof acquisition !== "string") return acquisition; + } + throw fail({ kind: "unrecoverable" }); + } + await ensureLeaseDirectory(input.directory, fail, produced); + const socketPath = join(input.directory, `${key}.sock`); + const acquisition = await tryAcquireLease(socketPath, owner, fail, true); + if (typeof acquisition !== "string") return acquisition; + + // Serialize stale cleanup through the replacement bind: otherwise two + // recoverers can unlink the socket that one of them has just acquired. + const recoveryPath = `${socketPath}.recovery`; + await reserveRecovery(recoveryPath, fail); + let recovered: LeaseAcquisition | "missing" | "stale" | undefined; + try { + recovered = await tryAcquireLease(socketPath, owner, fail, true); + if (recovered === "stale") await rm(socketPath, { force: true }); + if (typeof recovered === "string") + recovered = await tryAcquireLease(socketPath, owner, fail, true); + if (typeof recovered !== "string") return recovered; + throw fail({ kind: "unrecoverable" }); + } finally { + try { + await rmdir(recoveryPath); + } catch (cause: unknown) { + if (typeof recovered !== "string" && recovered?.acquired) + await recovered.lease.release(); + throw fail({ kind: "recovery_remove", path: recoveryPath }, cause); + } + } +}; + +const reserveRecovery = async ( + recoveryPath: string, + fail: (failure: LeaseFailure, cause?: unknown) => Error, +): Promise => { + try { + await mkdir(recoveryPath, { mode: 0o700 }); + } catch (cause: unknown) { + const busy = + typeof cause === "object" && + cause !== null && + Reflect.get(cause, "code") === "EEXIST"; + throw fail( + busy + ? { kind: "recovery_busy", path: recoveryPath } + : { kind: "recovery_create", path: recoveryPath }, + cause, + ); + } +}; + +const tryAcquireLease = async ( + socketPath: string, + owner: LeaseOwner, + fail: (failure: LeaseFailure, cause?: unknown) => Error, + securePath: boolean, +): Promise => { + const server = createServer((socket) => sendOwner(socket, owner)); + // The lease lives exactly as long as its owning process. A lease retained + // after unconfirmed cleanup must not keep a completed one-shot command + // alive; process exit leaves a stale socket that later acquirers recover. + server.unref(); + if (!(await bind(server, socketPath))) { + const existingOwner = await readOwner(socketPath, fail); + return typeof existingOwner === "string" + ? existingOwner + : { acquired: false, owner: existingOwner }; + } + if (securePath) + try { + await chmod(socketPath, 0o600); + } catch (cause: unknown) { + await closeServer(server); + throw cause; + } + let released = false; + return { + acquired: true, + lease: { + socketPath, + release: async () => { + if (released) return; + // Node removes a bound Unix socket on close. A second unlink here + // could remove a new owner's socket bound after that close. + await closeServer(server); + released = true; + }, + }, + }; +}; + +const ensureLeaseDirectory = async ( + directory: string, + fail: (failure: LeaseFailure, cause?: unknown) => Error, + produced: WeakSet, +): Promise => { + try { + await mkdir(directory, { recursive: true, mode: 0o700 }); + } catch (cause: unknown) { + throw fail({ kind: "directory_create" }, cause); + } + // Use O_DIRECTORY|O_NOFOLLOW to open the directory without following symlinks, + // then inspect and secure that descriptor. Later socket operations still use + // the path and are not made atomic by this directory validation. + let handle; + try { + handle = await open( + directory, + fsConstants.O_RDONLY | + (fsConstants.O_DIRECTORY ?? 0) | + (fsConstants.O_NOFOLLOW ?? 0), + ); + const directoryStat = await handle.stat(); + if ( + directoryStat.isSymbolicLink() || + directoryStat.isDirectory() === false || + (process.getuid !== undefined && directoryStat.uid !== process.getuid()) + ) + throw fail({ kind: "directory_not_private" }); + // POSIX permission hardening: chmod on the already-opened descriptor avoids + // a second TOCTOU window where the directory could be swapped between check + // and permission fix. + const posixPermissionsApply = process.getuid !== undefined; + if (posixPermissionsApply && (directoryStat.mode & 0o077) !== 0) { + await handle.chmod(0o700); + // Re-verify permissions on the same opened directory. + const afterChmod = await handle.stat(); + if ( + afterChmod.isSymbolicLink() || + afterChmod.isDirectory() === false || + (afterChmod.mode & 0o077) !== 0 + ) + throw fail({ kind: "directory_permissions" }); + } + } catch (cause: unknown) { + if (cause instanceof Error && produced.has(cause)) throw cause; + const accessDenied = + process.getuid !== undefined && + cause instanceof Error && + "code" in cause && + cause.code === "EACCES"; + throw fail( + accessDenied + ? { kind: "directory_access_denied", directory } + : { kind: "directory_inspect" }, + cause, + ); + } finally { + await handle?.close(); + } +}; + +const bind = (server: Server, socketPath: string): Promise => + new Promise((resolveBind, reject) => { + const onError = (error: NodeJS.ErrnoException): void => { + server.removeListener("listening", onListening); + if (error.code === "EADDRINUSE") resolveBind(false); + else reject(error); + }; + const onListening = (): void => { + server.removeListener("error", onError); + resolveBind(true); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(socketPath); + }); + +const sendOwner = (socket: Socket, owner: LeaseOwner): void => { + socket.end(`${JSON.stringify(owner)}\n`); +}; + +const readOwner = ( + socketPath: string, + fail: (failure: LeaseFailure, cause?: unknown) => Error, +): Promise => + new Promise((resolveOwner, rejectOwner) => { + const socket = createConnection(socketPath); + socket.setEncoding("utf8"); + let response = ""; + let settled = false; + const reject = (reason: string, cause?: unknown): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + rejectOwner(fail({ kind: "owner_unverified", reason }, cause)); + }; + const timer = setTimeout(() => reject("owner response timed out"), 250); + const finish = (owner: LeaseOwner | "missing" | "stale"): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + resolveOwner(owner); + }; + socket.on("data", (chunk: string) => { + response += chunk; + const newline = response.indexOf("\n"); + if (newline < 0) return; + const parsed = safeParseJson(response.slice(0, newline)); + if (!parsed.ok) { + reject("malformed owner response"); + return; + } + const value: unknown = parsed.value; + const runId = + typeof value === "object" && value !== null + ? Reflect.get(value, "runId") + : undefined; + const processId = + typeof value === "object" && value !== null + ? Reflect.get(value, "processId") + : undefined; + if ( + typeof runId === "string" && + runId.length > 0 && + typeof processId === "number" && + Number.isSafeInteger(processId) && + processId > 0 + ) + finish({ runId, processId }); + else reject("invalid owner identity"); + }); + socket.on("error", (cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT") finish("missing"); + else if (cause.code === "ECONNREFUSED") finish("stale"); + else reject(cause.code ?? "owner connection failed", cause); + }); + socket.on("end", () => reject("owner closed without a complete response")); + }); + +const closeServer = (server: Server): Promise => + new Promise((resolveClose, reject) => { + if (!server.listening) { + resolveClose(); + return; + } + server.close((error) => (error ? reject(error) : resolveClose())); + }); diff --git a/src/server/registerEnhancedTools.ts b/src/server/registerEnhancedTools.ts index 56ab29473d..2b3f29cbf4 100644 --- a/src/server/registerEnhancedTools.ts +++ b/src/server/registerEnhancedTools.ts @@ -15,6 +15,7 @@ import type { ProgressReporter } from "../application/ProgressReporter.js"; import { EnhancedTools } from "../application/EnhancedTools.js"; import type { ValidatedEnhancedCall } from "../application/EnhancedToolTypes.js"; import { + composedLimitationBudget, executeComposedWorkflow, createWorkflowEvidence, workflowSnapshotRecord, @@ -71,6 +72,7 @@ export const registerEnhancedTools = ( const objcClasses = toolContract("get_objc_classes"); const objcProtocols = toolContract("get_objc_protocols"); const batchDecompile = toolContract("batch_decompile"); + const decompileAll = toolContract("decompile_all"); const callGraph = toolContract("get_call_graph"); const swiftTypes = toolContract("analyze_swift_types"); const xrefsToName = toolContract("find_xrefs_to_name"); @@ -126,6 +128,15 @@ export const registerEnhancedTools = ( context, }), ); + server.registerTool( + decompileAll.name, + toolRegistrationOptions(decompileAll), + (input, context) => + executeEnhancedTool(registration, decompileAll, { + validatedCall: { name: "decompile_all", input }, + context, + }), + ); server.registerTool( callGraph.name, toolRegistrationOptions(callGraph), @@ -236,6 +247,7 @@ const executeEnhancedTool = async ( progress, }); const analysis: AnalysisOperationPort = { + resourceScope: admittedAnalysis.resourceScope, execute: (operation, operationParameters, executionOptions) => admittedAnalysis.execute(operation, operationParameters, { ...executionOptions, @@ -243,11 +255,14 @@ const executeEnhancedTool = async ( }), }; const result = await logToolExecution(registration.logger, name, () => - executeComposedWorkflow(analysis, (observed) => - new EnhancedTools(observed).executeValidated( - validatedCall, - context.mcpReq.signal, - ), + executeComposedWorkflow( + analysis, + (observed) => + new EnhancedTools(observed).executeValidated( + validatedCall, + context.mcpReq.signal, + ), + composedLimitationBudget(name), ), ); await reportOperationEnd(progress, name, result.ok); diff --git a/tests/boundary/cli/decompileAllCli.test.ts b/tests/boundary/cli/decompileAllCli.test.ts new file mode 100644 index 0000000000..bfb1934d89 --- /dev/null +++ b/tests/boundary/cli/decompileAllCli.test.ts @@ -0,0 +1,102 @@ +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; + +import { elf } from "../../../src/domain/binaryTarget.fixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { writeFakeKuna } from "../providers/kuna/fakeKuna.js"; + +const FUNCTIONS = [ + { name: "main", addr: "0x1000" }, + { name: "parse", addr: "0x1040" }, + { name: "late", addr: "0x1080" }, +] as const; + +describe("decompile-all CLI", () => { + cliTest( + "maps filters, limit and a relative output directory to the export", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-decompile-all-cli-"); + const kuna = await writeFakeKuna(root, { functions: FUNCTIONS }); + const target = join(root, "target.elf"); + await writeFile(target, elf(2, 1, 62)); + const environment = { + REA_LOG_LEVEL: "silent", + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: join(root, "cache"), + }; + const result = await cli.run({ + arguments: [ + "decompile-all", + target, + "--provider", + "kuna", + "--output", + "out", + "--address-start", + "0X1000", + "--address-end", + "0X1040", + "--limit", + "1", + "--format", + "json", + ], + cwd: root, + environment, + }); + expect(result.exitCode, JSON.stringify(result.json)).toBe(0); + expect(result.json).toMatchObject({ + operation: "decompile_all", + parameters: { + output_directory: join(root, "out"), + address_start: "0X1000", + address_end: "0X1040", + limit: 1, + layout: "address", + resume: false, + }, + normalized_result: { + output_directory: join(root, "out"), + // The fake inventory also lists the import puts at 0x2000. + counts: { selected: 2, excluded: 2, written: 1, remaining: 1 }, + }, + }); + expect(await readdir(join(root, "out", "functions"))).toEqual([ + "0x1000.c", + ]); + expect(JSON.stringify(result.json)).not.toContain( + await readFile(join(root, "out", "functions", "0x1000.c"), "utf8"), + ); + const resumed = await cli.run({ + arguments: [ + "decompile-all", + target, + "--provider", + "kuna", + "--output", + "out", + "--resume", + "--address-start", + "0X1000", + "--address-end", + "0X1040", + "--limit", + "100001", + "--format", + "json", + ], + cwd: root, + environment, + }); + expect(resumed.exitCode, JSON.stringify(resumed.json)).toBe(0); + expect(resumed.json).toMatchObject({ + parameters: { limit: 100001, resume: true }, + normalized_result: { + counts: { selected: 2, written: 1, skipped: 1, remaining: 0 }, + }, + }); + }, + ); +}); diff --git a/tests/boundary/filesystem/decompileExportCreate.test.ts b/tests/boundary/filesystem/decompileExportCreate.test.ts new file mode 100644 index 0000000000..732e5a24fc --- /dev/null +++ b/tests/boundary/filesystem/decompileExportCreate.test.ts @@ -0,0 +1,300 @@ +import { lstat, readFile, stat, writeFile, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +// Host filesystems cannot fail chmod on one new directory on demand; every +// other creation step runs on the real filesystem. +const failChmodFor = vi.hoisted(() => ({ + path: undefined as string | undefined, + journalClose: undefined as string | undefined, + beforeCleanup: undefined as (() => Promise) | undefined, + cleanupPath: undefined as string | undefined, + beforeLeasePath: undefined as string | undefined, + beforeLease: undefined as (() => Promise) | undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + if (args[0] === failChmodFor.journalClose) { + failChmodFor.journalClose = undefined; + const close = handle.close.bind(handle); + vi.spyOn(handle, "close") + .mockImplementationOnce(async () => { + throw new Error("injected close failure"); + }) + .mockImplementation(close); + } + return handle; + }, + rm: async (...args: Parameters) => { + if ( + args[0] === failChmodFor.cleanupPath || + `${String(args[0])}/decompile-journal.jsonl` === + failChmodFor.cleanupPath + ) { + const hook = failChmodFor.beforeCleanup; + failChmodFor.beforeCleanup = undefined; + await hook?.(); + } + return actual.rm(...args); + }, + unlink: async (...args: Parameters) => { + if (args[0] === failChmodFor.cleanupPath) { + const hook = failChmodFor.beforeCleanup; + failChmodFor.beforeCleanup = undefined; + await hook?.(); + } + return actual.unlink(...args); + }, + chmod: async (path: Parameters[0], mode: number) => { + if (path === failChmodFor.path) + throw Object.assign(new Error("denied"), { code: "EPERM" }); + await actual.chmod(path, mode); + if (path === failChmodFor.beforeLeasePath) { + const hook = failChmodFor.beforeLease; + failChmodFor.beforeLease = undefined; + await hook?.(); + } + }, + }; +}); + +const { DecompileExport } = + await import("../../../src/application/decompileDirectory/exportTree.js"); + +const { BinarySessionRecords } = + await import("../../../src/application/binary/BinarySessionRecords.js"); +class SessionResources extends BinarySessionRecords { + protected activeAnalysisBinding() { + return undefined; + } +} + +const { acquireOwnedLease } = + await import("../../../src/process/OwnedLease.js"); + +const identity = { + targetSha256: "ab".repeat(32), + provider: { id: "fixture", name: "Fixture", version: "1" }, + analysisProfileDigest: "cd".repeat(32), + layout: "address" as const, +}; + +afterEach(() => { + failChmodFor.path = undefined; + failChmodFor.journalClose = undefined; + failChmodFor.beforeCleanup = undefined; + failChmodFor.cleanupPath = undefined; + failChmodFor.beforeLeasePath = undefined; + failChmodFor.beforeLease = undefined; + vi.restoreAllMocks(); +}); + +describe("decompile export creation", () => { + it("removes the output directory it created when it cannot be made private", async () => { + const parent = await createTestTempDirectory("rea-decompile-create-"); + const root = join(parent, "out"); + failChmodFor.path = root; + + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + }); + + expect(opened.ok).toBe(false); + if (opened.ok) return; + expect(opened.error.userMessage).toContain("could not be secured"); + // A fresh export can be retried at the same path. + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + failChmodFor.path = undefined; + const retried = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + }); + expect(retried.ok).toBe(true); + if (retried.ok) expect((await retried.value.close()).ok).toBe(true); + }); + it("holds the lease while removing a failed initialization with a valid journal", async () => { + const parent = await createTestTempDirectory("rea-decompile-create-"); + const root = join(parent, "out"); + const journal = join(root, "decompile-journal.jsonl"); + failChmodFor.path = journal; + failChmodFor.cleanupPath = journal; + let resumedTag: string | undefined; + failChmodFor.beforeCleanup = async () => { + // The original header is valid when the competing run tries to resume. + expect( + JSON.parse((await readFile(journal, "utf8")).trim()), + ).toMatchObject({ kind: "header" }); + const resumed = await DecompileExport.open({ + placement: { mode: "resume", root }, + identity, + }); + if (resumed.ok) await resumed.value.close(); + resumedTag = resumed.ok ? "acquired" : resumed.error._tag; + }; + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + }); + expect(opened.ok).toBe(false); + expect(resumedTag).toBe("AnalysisResourceConstraintError"); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("retains an export after a journal close failure until session cleanup retries it", async () => { + const parent = await createTestTempDirectory("rea-decompile-close-"); + const root = join(parent, "out"); + const resources = new SessionResources(); + failChmodFor.journalClose = join(root, "decompile-journal.jsonl"); + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + resourceScope: resources.resourceScope, + }); + expect(opened.ok).toBe(true); + if (!opened.ok) return; + const closed = await opened.value.close(); + expect(closed.ok).toBe(false); + if (closed.ok) return; + expect(closed.error.cleanupIncomplete).toBe(true); + const refused = await DecompileExport.open({ + placement: { mode: "resume", root }, + identity, + }); + expect(refused.ok).toBe(false); + if (!refused.ok) + expect(refused.error._tag).toBe("AnalysisResourceConstraintError"); + expect((await resources.closeFileResources()).ok).toBe(true); + expect(closed.error.cleanupIncomplete).toBe(false); + const resumed = await DecompileExport.open({ + placement: { mode: "resume", root }, + identity, + }); + expect(resumed.ok).toBe(true); + if (resumed.ok) expect((await resumed.value.close()).ok).toBe(true); + }); + + it("preserves unexpected files and keeps initialization cleanup owned until a retry", async () => { + const parent = await createTestTempDirectory("rea-decompile-cleanup-"); + const root = join(parent, "out"); + const journal = join(root, "decompile-journal.jsonl"); + const resources = new SessionResources(); + failChmodFor.path = journal; + failChmodFor.cleanupPath = journal; + failChmodFor.beforeCleanup = () => + writeFile(join(root, "caller.txt"), "keep me"); + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + resourceScope: resources.resourceScope, + }); + expect(opened.ok).toBe(false); + if (opened.ok) return; + expect(opened.error.cleanupIncomplete).toBe(true); + expect(await readFile(join(root, "caller.txt"), "utf8")).toBe("keep me"); + expect((await resources.closeFileResources()).ok).toBe(false); + await unlink(join(root, "caller.txt")); + expect((await resources.closeFileResources()).ok).toBe(true); + expect(opened.error.cleanupIncomplete).toBe(false); + await expect(stat(root)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + +describe("decompile export lease acquisition", () => { + it("preserves an empty new root when another run acquires its lease first", async () => { + const parent = await createTestTempDirectory("rea-decompile-lease-race-"); + const root = join(parent, "out"); + let release: (() => Promise) | undefined; + failChmodFor.beforeLeasePath = root; + failChmodFor.beforeLease = async () => { + const stat = await lstat(root, { bigint: true }); + const acquired = await acquireOwnedLease({ + identity: { + operation: "decompile_all", + device: stat.dev.toString(), + inode: stat.ino.toString(), + }, + runId: "competing-export", + directory: join("/tmp", `rea-decompile-${process.getuid?.() ?? 0}`), + failure: () => new Error("competing lease could not be acquired"), + }); + expect(acquired.acquired).toBe(true); + if (acquired.acquired) release = acquired.lease.release; + }; + try { + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + }); + expect(opened.ok).toBe(false); + expect((await stat(root)).isDirectory()).toBe(true); + if (!opened.ok) { + expect(opened.error._tag).toBe("AnalysisResourceConstraintError"); + expect(opened.error.diagnostics).toMatchObject({ + created_output_directory: root, + }); + } + } finally { + await release?.(); + } + }); +}); + +describe("decompile journal ownership", () => { + it("keeps the parsed committed record when a caller mutates its input during append", async () => { + const parent = await createTestTempDirectory("rea-decompile-record-"); + const root = join(parent, "out"); + const opened = await DecompileExport.open({ + placement: { mode: "create", root }, + identity, + }); + expect(opened.ok).toBe(true); + if (!opened.ok) return; + try { + const record = { + address: "0x1000", + name: "original", + status: "failed" as const, + body: { available: false as const, reason: "original body" }, + failure: { code: "fixture", message: "original failure" }, + limitations: ["original caveat"], + }; + const appended = opened.value.append(record); + record.address = "0x2000"; + record.body.reason = "changed body"; + record.failure.message = "changed failure"; + record.limitations[0] = "changed caveat"; + expect((await appended).ok).toBe(true); + expect( + ( + await opened.value.writeIndex({ + layoutStatus: "address", + limitations: [], + }) + ).ok, + ).toBe(true); + const index = JSON.parse( + await readFile(join(root, "decompile-index.json"), "utf8"), + ); + expect(index.functions).toEqual([ + { + address: "0x1000", + name: "original", + status: "failed", + body: { available: false, reason: "original body" }, + failure: { code: "fixture", message: "original failure" }, + limitations: ["original caveat"], + }, + ]); + } finally { + expect((await opened.value.close()).ok).toBe(true); + } + }); +}); diff --git a/tests/boundary/mcp/decompileAllCaveats.test.ts b/tests/boundary/mcp/decompileAllCaveats.test.ts new file mode 100644 index 0000000000..cc3b5987a7 --- /dev/null +++ b/tests/boundary/mcp/decompileAllCaveats.test.ts @@ -0,0 +1,146 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + createAnalysisExecution, + type AnalysisOperationPort, +} from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { err, ok } from "../../../src/domain/result.js"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { + decompileDirectoryIndexSchema, + decompileDirectoryManifestSchema, +} from "../../../src/contracts/decompileDirectory.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; + +const provider = { id: "fixture", name: "Fixture", version: "1" }; +const profile = createAnalysisProfile(provider, { decompiler: "fixture" }); +const addresses = Array.from( + { length: 20 }, + (_, index) => `0x${(0x1000 + index * 0x10).toString(16)}`, +); + +const longCaveat = "long provider caveat ".repeat(600); + +/** A provider that, like Ghidra, names each function's terminal call address. */ +const addressCaveats: AnalysisOperationPort = { + execute: (operation, parameters) => { + const observe = (value: unknown, limitations: readonly string[] = []) => + Promise.resolve( + ok( + createAnalysisExecution(value, provider, { + analysisProfile: profile, + limitations, + }), + ), + ); + const address = String(parameters.procedure); + if (operation === "list_procedures") + return observe( + addresses.map((entry) => ({ address: entry, value: entry })), + ); + if (operation === "procedure_info") + return observe(null, [`procedure_info caveat at ${address}`]); + return observe(`int f(void) { return 0; }`, [ + `terminal call at ${address}`, + ...(address === addresses.at(-1) ? [longCaveat] : []), + ]); + }, +}; + +afterEach(closeEnhancedToolResources); + +describe("decompile_all MCP caveats", () => { + it("returns a bounded caveat summary and keeps each function's caveats in its file", async () => { + const parent = await createTestTempDirectory("rea-decompile-caveats-"); + const client = await connect(addressCaveats); + const result = await client.callTool({ + name: "decompile_all", + arguments: { output_directory: join(parent, "out") }, + }); + expect(result.isError).not.toBe(true); + const evidence = parseEvidence(result.structuredContent); + expect(evidence.normalized_result).toMatchObject({ + counts: { written: 20, failed: 0 }, + }); + // 40 address-specific caveats plus one oversized caveat: 8 kept and 33 counted, beside the workflow + // line and this target-less fixture's identity caveat. + expect(evidence.limitations).toHaveLength(11); + expect(evidence.limitations.at(-1)).toMatch( + /^33 per-item provider limitation observations were omitted/u, + ); + const last = await readFile( + join(parent, "out", "functions", `${addresses.at(-1)}.c`), + "utf8", + ); + expect(last).toContain(`procedure_info caveat at ${addresses.at(-1)}`); + expect(last).toContain(`terminal call at ${addresses.at(-1)}`); + expect(JSON.stringify(result)).not.toContain(longCaveat); + const index = JSON.parse( + await readFile(join(parent, "out", "decompile-index.json"), "utf8"), + ); + expect(index.functions.at(-1).limitations).toContain(longCaveat); + }); +}); + +it("keeps long-name failures inside the tool window and complete in the index", async () => { + const parent = await createTestTempDirectory("rea-decompile-long-failures-"); + const output = join(parent, "out"); + const names = Array.from( + { length: 32 }, + (_, index) => `${'a"\\\0😀'.repeat(400)}-${index}`, + ); + const caveat = "\0".repeat(2_000); + const port: AnalysisOperationPort = { + execute: async (operation) => + operation === "procedure_pseudo_code" + ? err(new AnalysisInputError(operation)) + : ok( + createAnalysisExecution( + operation === "list_procedures" + ? names.map((name, index) => ({ + address: `0x${(0x1000 + index).toString(16)}`, + value: name, + })) + : null, + provider, + { + analysisProfile: profile, + ...(operation === "procedure_info" + ? { limitations: [caveat] } + : {}), + }, + ), + ), + }; + const client = await connect(port); + const result = await client.callTool({ + name: "decompile_all", + arguments: { output_directory: output }, + }); + expect(result.isError).not.toBe(true); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(50 * 1_024); + const evidence = parseEvidence(result.structuredContent); + const manifest = decompileDirectoryManifestSchema.parse( + evidence.normalized_result, + ); + expect(manifest.counts.failed).toBe(32); + expect(manifest.failures.length).toBeGreaterThan(0); + expect(manifest.failures_omitted + manifest.failures.length).toBe(32); + expect(manifest.failures.every((failure) => !("name" in failure))).toBe(true); + expect(manifest.failures.every((failure) => failure.code.length > 0)).toBe( + true, + ); + expect(JSON.stringify(result)).not.toContain(caveat); + const index = decompileDirectoryIndexSchema.parse( + JSON.parse(await readFile(join(output, "decompile-index.json"), "utf8")), + ); + expect(index.functions.map((record) => record.name)).toEqual(names); + expect( + index.functions.every((record) => record.limitations?.includes(caveat)), + ).toBe(true); +}); diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index d260d927b2..0de734f2ed 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -1,3 +1,6 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { afterEach, describe, expect, it } from "vitest"; import { PROCEDURES, inventory, jsonResult } from "./enhancedToolsHarness.js"; @@ -63,29 +66,24 @@ const functionDossier = { limitations: [], }; +const capabilityUnavailable = (operation: string, message: string) => + Promise.resolve( + err(new AnalysisCapabilityUnavailableError("fixture", operation, message)), + ); + const fixturePort = (): AnalysisOperationPort => ({ // oxlint-disable-next-line complexity -- this fixture exhaustively serves the registered MCP surface. execute: (name, arguments_) => { switch (name) { case "read_binary_inventory_summary": - return Promise.resolve( - err( - new AnalysisCapabilityUnavailableError( - "fixture", - name, - "This fixture exercises composed overview", - ), - ), + return capabilityUnavailable( + name, + "This fixture exercises composed overview", ); case "inspect_native_dispatch_metadata": - return Promise.resolve( - err( - new AnalysisCapabilityUnavailableError( - "fixture", - name, - "No byte reader or exact native seed in this symbol-only fixture", - ), - ), + return capabilityUnavailable( + name, + "No byte reader or exact native seed in this symbol-only fixture", ); case "procedure_address": { const procedure = inventory(PROCEDURES).find( @@ -153,6 +151,11 @@ const fixturePort = (): AnalysisOperationPort => ({ ok(typeof procedure === "string" ? `pseudo:${procedure}` : "invalid"), ); } + case "procedure_info": + return capabilityUnavailable( + name, + "This fixture does not report function bodies", + ); case "procedure_callees": { const procedure = arguments_.procedure; return Promise.resolve( @@ -215,11 +218,16 @@ const fixturePort = (): AnalysisOperationPort => ({ }); const resources: Array<{ close(): Promise }> = []; +let decompileOutput: string | undefined; afterEach(async () => { await Promise.all( resources.splice(0).map(async (resource) => resource.close()), ); + if (decompileOutput !== undefined) { + await rm(decompileOutput, { recursive: true, force: true }); + decompileOutput = undefined; + } }); const connect = async (analysis: AnalysisOperationPort = fixturePort()) => { @@ -257,6 +265,8 @@ describe("enhanced MCP tools", () => { // oxlint-disable-next-line max-lines-per-function -- one exhaustive registration test guards the public tool catalog. it("executes all enhanced tools through production registration", async () => { const client = await connect(); + const outputParent = await mkdtemp(join(tmpdir(), "rea-decompile-all-")); + decompileOutput = outputParent; const calls = [ ["get_objc_classes", { pattern: "Fixture" }], ["get_objc_protocols", {}], @@ -272,6 +282,7 @@ describe("enhanced MCP tools", () => { ["trace_native_ui_action", { action: "missing-selector" }], ["inspect_native_dispatch_metadata", { max_records: 100 }], ["trace_native_values", { procedure: "0x1" }], + ["decompile_all", { output_directory: join(outputParent, "out") }], ] as const; const results = await Promise.all( calls.map(async ([name, arguments_]) => @@ -438,6 +449,11 @@ describe("enhanced MCP tools", () => { }), ], }); + expect(results[14]).toMatchObject({ + counts: { selected: 6, written: 6, failed: 0, excluded: 0 }, + layout_status: "address", + }); + expect(JSON.stringify(results[14])).not.toContain("pseudo:"); }); });